@elizaos/plugin-xr 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/actions/xr-query-vision.d.ts

@@ -0,0 +1,3 @@
+import type { Action } from "@elizaos/core";
+export declare const xrQueryVisionAction: Action;
+//# sourceMappingURL=xr-query-vision.d.ts.map

package/dist/actions/xr-query-vision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"xr-query-vision.d.ts","sourceRoot":"","sources":["../../src/actions/xr-query-vision.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAMP,MAAM,eAAe,CAAC;AAMvB,eAAO,MAAM,mBAAmB,EAAE,MAkDjC,CAAC"}

package/dist/actions/xr-query-vision.js

@@ -0,0 +1,39 @@
+import {
+  XR_SERVICE_TYPE
+} from "../services/xr-session-service.js";
+const xrQueryVisionAction = {
+  name: "XR_QUERY_VISION",
+  description: "Describe what the user is currently looking at through their XR headset camera. Use this when the user asks 'what do you see', 'look at this', or any question about their surroundings.",
+  validate: async (runtime) => {
+    const svc = runtime.getService(XR_SERVICE_TYPE);
+    if (!svc?.hasActiveConnections()) return false;
+    return svc.getConnections().some((c) => svc.getVisionPipeline().hasRecentFrame(c.id));
+  },
+  handler: async (runtime, message, _state, _options, callback) => {
+    const svc = runtime.getService(XR_SERVICE_TYPE);
+    if (!svc) {
+      await callback?.({ text: "XR service is not running." });
+      return void 0;
+    }
+    const conn = svc.getConnections().find((c) => c.roomId === message.roomId);
+    if (!conn) {
+      await callback?.({
+        text: "No XR device connection found for this session."
+      });
+      return void 0;
+    }
+    const description = await svc.getVisionPipeline().describeFrame(runtime, conn.id);
+    if (!description) {
+      await callback?.({
+        text: "No recent camera frame available from the XR device."
+      });
+      return void 0;
+    }
+    await callback?.({ text: description });
+    return void 0;
+  }
+};
+export {
+  xrQueryVisionAction
+};
+//# sourceMappingURL=xr-query-vision.js.map

package/dist/actions/xr-query-vision.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/actions/xr-query-vision.ts"],"sourcesContent":["import type {\n  Action,\n  ActionResult,\n  HandlerCallback,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport {\n  XR_SERVICE_TYPE,\n  type XRSessionService,\n} from \"../services/xr-session-service.js\";\n\nexport const xrQueryVisionAction: Action = {\n  name: \"XR_QUERY_VISION\",\n  description:\n    \"Describe what the user is currently looking at through their XR headset camera. Use this when the user asks 'what do you see', 'look at this', or any question about their surroundings.\",\n\n  validate: async (runtime: IAgentRuntime): Promise<boolean> => {\n    const svc = runtime.getService<XRSessionService>(XR_SERVICE_TYPE);\n    if (!svc?.hasActiveConnections()) return false;\n    // At least one connection must have a recent frame\n    return svc\n      .getConnections()\n      .some((c) => svc.getVisionPipeline().hasRecentFrame(c.id));\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state: State | undefined,\n    _options: unknown,\n    callback?: HandlerCallback,\n  ): Promise<ActionResult | undefined> => {\n    const svc = runtime.getService<XRSessionService>(XR_SERVICE_TYPE);\n    if (!svc) {\n      await callback?.({ text: \"XR service is not running.\" });\n      return undefined;\n    }\n\n    const conn = svc.getConnections().find((c) => c.roomId === message.roomId);\n\n    if (!conn) {\n      await callback?.({\n        text: \"No XR device connection found for this session.\",\n      });\n      return undefined;\n    }\n\n    const description = await svc\n      .getVisionPipeline()\n      .describeFrame(runtime, conn.id);\n\n    if (!description) {\n      await callback?.({\n        text: \"No recent camera frame available from the XR device.\",\n      });\n      return undefined;\n    }\n\n    await callback?.({ text: description });\n    return undefined;\n  },\n};\n"],"mappings":"AAQA;AAAA,EACE;AAAA,OAEK;AAEA,MAAM,sBAA8B;AAAA,EACzC,MAAM;AAAA,EACN,aACE;AAAA,EAEF,UAAU,OAAO,YAA6C;AAC5D,UAAM,MAAM,QAAQ,WAA6B,eAAe;AAChE,QAAI,CAAC,KAAK,qBAAqB,EAAG,QAAO;AAEzC,WAAO,IACJ,eAAe,EACf,KAAK,CAAC,MAAM,IAAI,kBAAkB,EAAE,eAAe,EAAE,EAAE,CAAC;AAAA,EAC7D;AAAA,EAEA,SAAS,OACP,SACA,SACA,QACA,UACA,aACsC;AACtC,UAAM,MAAM,QAAQ,WAA6B,eAAe;AAChE,QAAI,CAAC,KAAK;AACR,YAAM,WAAW,EAAE,MAAM,6BAA6B,CAAC;AACvD,aAAO;AAAA,IACT;AAEA,UAAM,OAAO,IAAI,eAAe,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,QAAQ,MAAM;AAEzE,QAAI,CAAC,MAAM;AACT,YAAM,WAAW;AAAA,QACf,MAAM;AAAA,MACR,CAAC;AACD,aAAO;AAAA,IACT;AAEA,UAAM,cAAc,MAAM,IACvB,kBAAkB,EAClB,cAAc,SAAS,KAAK,EAAE;AAEjC,QAAI,CAAC,aAAa;AAChB,YAAM,WAAW;AAAA,QACf,MAAM;AAAA,MACR,CAAC;AACD,aAAO;AAAA,IACT;AAEA,UAAM,WAAW,EAAE,MAAM,YAAY,CAAC;AACtC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/actions/xr-view-actions.d.ts

@@ -0,0 +1,18 @@
+import type { Action } from "@elizaos/core";
+export declare const xrOpenViewAction: Action;
+export declare const xrCloseViewAction: Action;
+export declare const xrSwitchViewAction: Action;
+export declare const xrListViewsAction: Action;
+export declare const xrResizeViewAction: Action;
+/** Extract a likely view id from natural language */
+export declare function extractViewId(text: string): string;
+type XRViewSummary = {
+    id: string;
+    label: string;
+    icon?: string;
+    description?: string;
+};
+/** Collect all XR-typed views from registered plugins */
+export declare function collectXRViews(): XRViewSummary[];
+export {};
+//# sourceMappingURL=xr-view-actions.d.ts.map

package/dist/actions/xr-view-actions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"xr-view-actions.d.ts","sourceRoot":"","sources":["../../src/actions/xr-view-actions.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,MAAM,EAKP,MAAM,eAAe,CAAC;AA4BvB,eAAO,MAAM,gBAAgB,EAAE,MAoE9B,CAAC;AAIF,eAAO,MAAM,iBAAiB,EAAE,MAqD/B,CAAC;AAIF,eAAO,MAAM,kBAAkB,EAAE,MA4ChC,CAAC;AAIF,eAAO,MAAM,iBAAiB,EAAE,MAkD/B,CAAC;AAIF,eAAO,MAAM,kBAAkB,EAAE,MA6EhC,CAAC;AAIF,qDAAqD;AACrD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAalD;AAED,KAAK,aAAa,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,yDAAyD;AACzD,wBAAgB,cAAc,IAAI,aAAa,EAAE,CAchD"}

package/dist/actions/xr-view-actions.js

@@ -0,0 +1,304 @@
+import { listViews } from "@elizaos/agent/api/views-registry";
+import {
+  XR_SERVICE_TYPE
+} from "../services/xr-session-service.js";
+function getService(runtime) {
+  return runtime.getService(XR_SERVICE_TYPE) ?? null;
+}
+function firstConnectionId(svc) {
+  return svc.getConnections()[0]?.id ?? null;
+}
+function agentBaseUrl() {
+  const port = process.env.ELIZA_API_PORT?.trim() || process.env.ELIZA_PORT?.trim() || "31337";
+  return process.env.XR_AGENT_URL ?? `http://localhost:${port}`;
+}
+const xrOpenViewAction = {
+  name: "XR_OPEN_VIEW",
+  similes: ["OPEN_XR_VIEW", "SHOW_XR_PANEL", "XR_SHOW", "XR_LAUNCH"],
+  description: "Opens a view panel on the connected XR headset by view id. Use XR_LIST_VIEWS first to discover available view ids.",
+  examples: [
+    [
+      { name: "user", content: { text: "open the wallet in XR" } },
+      {
+        name: "agent",
+        content: {
+          text: "Opening wallet view on your headset.",
+          action: "XR_OPEN_VIEW"
+        }
+      }
+    ],
+    [
+      { name: "user", content: { text: "show training dashboard in XR" } },
+      {
+        name: "agent",
+        content: { text: "Launching training panel.", action: "XR_OPEN_VIEW" }
+      }
+    ]
+  ],
+  validate: async (runtime) => {
+    const svc = getService(runtime);
+    return svc?.hasActiveConnections() ?? false;
+  },
+  handler: async (runtime, message, _state, options, callback) => {
+    const svc = getService(runtime);
+    if (!svc) {
+      const text2 = "No XR session service available.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const viewId = options?.viewId ?? extractViewId(message.content.text ?? "");
+    if (!viewId) {
+      const text2 = "Please specify which view to open. Try XR_LIST_VIEWS to see available views.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const connId = firstConnectionId(svc);
+    if (!connId) {
+      const text2 = "No XR device is connected.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const base = agentBaseUrl();
+    const scale = options?.scale ?? 1;
+    svc.openView(connId, viewId, base, { scale, followMode: "billboard" });
+    const text = `Opening ${viewId} view on your headset.`;
+    await callback?.({ text });
+    return { success: true, text };
+  }
+};
+const xrCloseViewAction = {
+  name: "XR_CLOSE_VIEW",
+  similes: ["CLOSE_XR_VIEW", "HIDE_XR_PANEL", "XR_CLOSE", "XR_DISMISS"],
+  description: "Closes a specific view panel on the connected XR headset.",
+  examples: [
+    [
+      { name: "user", content: { text: "close the wallet panel" } },
+      {
+        name: "agent",
+        content: { text: "Closing wallet panel.", action: "XR_CLOSE_VIEW" }
+      }
+    ]
+  ],
+  validate: async (runtime) => {
+    const svc = getService(runtime);
+    return svc?.hasActiveConnections() ?? false;
+  },
+  handler: async (runtime, message, _state, options, callback) => {
+    const svc = getService(runtime);
+    if (!svc) {
+      const text2 = "No XR service.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const viewId = options?.viewId ?? extractViewId(message.content.text ?? "");
+    const connId = firstConnectionId(svc);
+    if (!connId) {
+      const text2 = "No XR device connected.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    let text;
+    if (viewId) {
+      svc.closeView(connId, viewId);
+      text = `Closed ${viewId}.`;
+      await callback?.({ text });
+    } else {
+      const views = collectXRViews();
+      for (const view of views) {
+        svc.closeView(connId, view.id);
+      }
+      text = views.length > 0 ? "Closed all XR panels." : "No XR panels to close.";
+      await callback?.({ text });
+    }
+    return { success: true, text };
+  }
+};
+const xrSwitchViewAction = {
+  name: "XR_SWITCH_VIEW",
+  similes: ["SWITCH_XR_VIEW", "XR_GO_TO", "XR_NAVIGATE"],
+  description: "Switches the active (foreground) view on the XR headset without closing others.",
+  examples: [
+    [
+      { name: "user", content: { text: "switch to companion in XR" } },
+      {
+        name: "agent",
+        content: {
+          text: "Switching to companion view.",
+          action: "XR_SWITCH_VIEW"
+        }
+      }
+    ]
+  ],
+  validate: async (runtime) => {
+    const svc = getService(runtime);
+    return svc?.hasActiveConnections() ?? false;
+  },
+  handler: async (runtime, message, _state, options, callback) => {
+    const svc = getService(runtime);
+    if (!svc) {
+      const text2 = "No XR service.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const viewId = options?.viewId ?? extractViewId(message.content.text ?? "");
+    const connId = firstConnectionId(svc);
+    if (!connId || !viewId) {
+      const text2 = "Specify a view id.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    svc.switchView(connId, viewId);
+    const text = `Switched to ${viewId}.`;
+    await callback?.({ text });
+    return { success: true, text };
+  }
+};
+const xrListViewsAction = {
+  name: "XR_LIST_VIEWS",
+  similes: ["LIST_XR_VIEWS", "XR_VIEWS", "WHAT_XR_VIEWS", "SHOW_XR_LAUNCHER"],
+  description: "Lists all views available on the XR device and optionally sends a launcher catalog to the headset. Use this before XR_OPEN_VIEW.",
+  examples: [
+    [
+      { name: "user", content: { text: "what can I open in XR?" } },
+      {
+        name: "agent",
+        content: {
+          text: "Available XR views are registered by the loaded plugins.",
+          action: "XR_LIST_VIEWS"
+        }
+      }
+    ]
+  ],
+  validate: async (runtime) => {
+    const svc = getService(runtime);
+    return svc !== null;
+  },
+  handler: async (runtime, _message, _state, options, callback) => {
+    const svc = getService(runtime);
+    if (!svc) {
+      const text2 = "No XR service.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const xrViews = collectXRViews();
+    const connId = firstConnectionId(svc);
+    if (connId && options?.sendCatalog !== false) {
+      svc.sendViewsCatalog(connId, xrViews);
+    }
+    if (xrViews.length === 0) {
+      const text2 = "No XR views are currently registered.";
+      await callback?.({ text: text2 });
+      return { success: true, text: text2 };
+    }
+    const list = xrViews.map((v) => `\u2022 ${v.label} (id: ${v.id})`).join("\n");
+    const text = `Available XR views:
+${list}
+
+Say "open [view name]" to launch one.`;
+    await callback?.({ text });
+    return { success: true, text };
+  }
+};
+const xrResizeViewAction = {
+  name: "XR_RESIZE_VIEW",
+  similes: ["RESIZE_XR_PANEL", "XR_MAKE_BIGGER", "XR_MAKE_SMALLER", "XR_SCALE"],
+  description: "Resizes or repositions the active XR view panel. Accepts scale (0.5 = half, 2.0 = double) and distance.",
+  examples: [
+    [
+      { name: "user", content: { text: "make the panel bigger" } },
+      {
+        name: "agent",
+        content: { text: "Scaling up the panel.", action: "XR_RESIZE_VIEW" }
+      }
+    ],
+    [
+      { name: "user", content: { text: "make it smaller and move closer" } },
+      {
+        name: "agent",
+        content: {
+          text: "Resizing and moving panel closer.",
+          action: "XR_RESIZE_VIEW"
+        }
+      }
+    ]
+  ],
+  validate: async (runtime) => {
+    const svc = getService(runtime);
+    return svc?.hasActiveConnections() ?? false;
+  },
+  handler: async (runtime, message, _state, options, callback) => {
+    const svc = getService(runtime);
+    if (!svc) {
+      const text2 = "No XR service.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const connId = firstConnectionId(svc);
+    if (!connId) {
+      const text2 = "No XR device connected.";
+      await callback?.({ text: text2 });
+      return { success: false, text: text2 };
+    }
+    const inputText = message.content.text?.toLowerCase() ?? "";
+    let scale = options?.scale ?? 1;
+    let distance = options?.distance ?? 1.5;
+    if (inputText.includes("bigger") || inputText.includes("larger") || inputText.includes("bigger"))
+      scale = 1.5;
+    if (inputText.includes("smaller") || inputText.includes("tiny"))
+      scale = 0.6;
+    if (inputText.includes("closer") || inputText.includes("nearer"))
+      distance = 0.8;
+    if (inputText.includes("farther") || inputText.includes("further") || inputText.includes("away"))
+      distance = 2.5;
+    if (inputText.includes("fullscreen") || inputText.includes("full screen")) {
+      svc.resizeView(connId, "", { scale: 2, fullscreen: true });
+      const text2 = "Panel fullscreened.";
+      await callback?.({ text: text2 });
+      return { success: true, text: text2 };
+    }
+    const viewId = options?.viewId ?? "";
+    svc.resizeView(connId, viewId, { scale, distance });
+    const text = `Panel resized to ${scale}\xD7 at ${distance}m.`;
+    await callback?.({ text });
+    return { success: true, text };
+  }
+};
+function extractViewId(text) {
+  const lower = text.toLowerCase();
+  for (const view of collectXRViews()) {
+    const terms = [view.id, view.id.replace(/-/g, " "), view.label].map(
+      (term) => term.toLowerCase()
+    );
+    if (terms.some((term) => term && lower.includes(term))) {
+      return view.id;
+    }
+  }
+  const quoted = text.match(/["']([^"']+)["']/);
+  if (quoted?.[1]) return quoted[1];
+  return "";
+}
+function collectXRViews() {
+  const byId = /* @__PURE__ */ new Map();
+  for (const view of listViews({ developerMode: true, viewType: "xr" })) {
+    if (view.viewType !== "xr") continue;
+    byId.set(view.id, {
+      id: view.id,
+      label: view.label,
+      icon: view.icon,
+      description: view.description
+    });
+  }
+  return [...byId.values()].sort(
+    (a, b) => a.label.localeCompare(b.label) || a.id.localeCompare(b.id)
+  );
+}
+export {
+  collectXRViews,
+  extractViewId,
+  xrCloseViewAction,
+  xrListViewsAction,
+  xrOpenViewAction,
+  xrResizeViewAction,
+  xrSwitchViewAction
+};
+//# sourceMappingURL=xr-view-actions.js.map

package/dist/actions/xr-view-actions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/actions/xr-view-actions.ts"],"sourcesContent":["import { listViews } from \"@elizaos/agent/api/views-registry\";\nimport type {\n  Action,\n  HandlerCallback,\n  IAgentRuntime,\n  Memory,\n  State,\n} from \"@elizaos/core\";\nimport {\n  XR_SERVICE_TYPE,\n  type XRSessionService,\n} from \"../services/xr-session-service.js\";\n\n// ── Helpers ────────────────────────────────────────────────────────────────\n\nfunction getService(runtime: IAgentRuntime): XRSessionService | null {\n  return runtime.getService<XRSessionService>(XR_SERVICE_TYPE) ?? null;\n}\n\nfunction firstConnectionId(svc: XRSessionService): string | null {\n  return svc.getConnections()[0]?.id ?? null;\n}\n\nfunction agentBaseUrl(): string {\n  // The API port is orchestrator-assigned (never hardcoded) — read it from the\n  // environment, not a non-existent `runtime.port` field. Falls back to 31337.\n  const port =\n    process.env.ELIZA_API_PORT?.trim() ||\n    process.env.ELIZA_PORT?.trim() ||\n    \"31337\";\n  return process.env.XR_AGENT_URL ?? `http://localhost:${port}`;\n}\n\n// ── XR_OPEN_VIEW ───────────────────────────────────────────────────────────\n\nexport const xrOpenViewAction: Action = {\n  name: \"XR_OPEN_VIEW\",\n  similes: [\"OPEN_XR_VIEW\", \"SHOW_XR_PANEL\", \"XR_SHOW\", \"XR_LAUNCH\"],\n  description:\n    \"Opens a view panel on the connected XR headset by view id. Use XR_LIST_VIEWS first to discover available view ids.\",\n  examples: [\n    [\n      { name: \"user\", content: { text: \"open the wallet in XR\" } },\n      {\n        name: \"agent\",\n        content: {\n          text: \"Opening wallet view on your headset.\",\n          action: \"XR_OPEN_VIEW\",\n        },\n      },\n    ],\n    [\n      { name: \"user\", content: { text: \"show training dashboard in XR\" } },\n      {\n        name: \"agent\",\n        content: { text: \"Launching training panel.\", action: \"XR_OPEN_VIEW\" },\n      },\n    ],\n  ],\n\n  validate: async (runtime): Promise<boolean> => {\n    const svc = getService(runtime);\n    return svc?.hasActiveConnections() ?? false;\n  },\n\n  handler: async (\n    runtime: IAgentRuntime,\n    message: Memory,\n    _state: State | undefined,\n    options: Record<string, unknown> | undefined,\n    callback?: HandlerCallback,\n  ) => {\n    const svc = getService(runtime);\n    if (!svc) {\n      const text = \"No XR session service available.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    const viewId =\n      (options?.viewId as string | undefined) ??\n      extractViewId(message.content.text ?? \"\");\n    if (!viewId) {\n      const text =\n        \"Please specify which view to open. Try XR_LIST_VIEWS to see available views.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    const connId = firstConnectionId(svc);\n    if (!connId) {\n      const text = \"No XR device is connected.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    const base = agentBaseUrl();\n    const scale = (options?.scale as number | undefined) ?? 1.0;\n    svc.openView(connId, viewId, base, { scale, followMode: \"billboard\" });\n    const text = `Opening ${viewId} view on your headset.`;\n    await callback?.({ text });\n    return { success: true, text };\n  },\n};\n\n// ── XR_CLOSE_VIEW ──────────────────────────────────────────────────────────\n\nexport const xrCloseViewAction: Action = {\n  name: \"XR_CLOSE_VIEW\",\n  similes: [\"CLOSE_XR_VIEW\", \"HIDE_XR_PANEL\", \"XR_CLOSE\", \"XR_DISMISS\"],\n  description: \"Closes a specific view panel on the connected XR headset.\",\n  examples: [\n    [\n      { name: \"user\", content: { text: \"close the wallet panel\" } },\n      {\n        name: \"agent\",\n        content: { text: \"Closing wallet panel.\", action: \"XR_CLOSE_VIEW\" },\n      },\n    ],\n  ],\n\n  validate: async (runtime) => {\n    const svc = getService(runtime);\n    return svc?.hasActiveConnections() ?? false;\n  },\n\n  handler: async (runtime, message, _state, options, callback) => {\n    const svc = getService(runtime);\n    if (!svc) {\n      const text = \"No XR service.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    const viewId =\n      (options?.viewId as string | undefined) ??\n      extractViewId(message.content.text ?? \"\");\n    const connId = firstConnectionId(svc);\n    if (!connId) {\n      const text = \"No XR device connected.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    let text: string;\n    if (viewId) {\n      svc.closeView(connId, viewId);\n      text = `Closed ${viewId}.`;\n      await callback?.({ text });\n    } else {\n      const views = collectXRViews();\n      for (const view of views) {\n        svc.closeView(connId, view.id);\n      }\n      text =\n        views.length > 0 ? \"Closed all XR panels.\" : \"No XR panels to close.\";\n      await callback?.({ text });\n    }\n    return { success: true, text };\n  },\n};\n\n// ── XR_SWITCH_VIEW ─────────────────────────────────────────────────────────\n\nexport const xrSwitchViewAction: Action = {\n  name: \"XR_SWITCH_VIEW\",\n  similes: [\"SWITCH_XR_VIEW\", \"XR_GO_TO\", \"XR_NAVIGATE\"],\n  description:\n    \"Switches the active (foreground) view on the XR headset without closing others.\",\n  examples: [\n    [\n      { name: \"user\", content: { text: \"switch to companion in XR\" } },\n      {\n        name: \"agent\",\n        content: {\n          text: \"Switching to companion view.\",\n          action: \"XR_SWITCH_VIEW\",\n        },\n      },\n    ],\n  ],\n\n  validate: async (runtime) => {\n    const svc = getService(runtime);\n    return svc?.hasActiveConnections() ?? false;\n  },\n\n  handler: async (runtime, message, _state, options, callback) => {\n    const svc = getService(runtime);\n    if (!svc) {\n      const text = \"No XR service.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n    const viewId =\n      (options?.viewId as string | undefined) ??\n      extractViewId(message.content.text ?? \"\");\n    const connId = firstConnectionId(svc);\n    if (!connId || !viewId) {\n      const text = \"Specify a view id.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n    svc.switchView(connId, viewId);\n    const text = `Switched to ${viewId}.`;\n    await callback?.({ text });\n    return { success: true, text };\n  },\n};\n\n// ── XR_LIST_VIEWS ──────────────────────────────────────────────────────────\n\nexport const xrListViewsAction: Action = {\n  name: \"XR_LIST_VIEWS\",\n  similes: [\"LIST_XR_VIEWS\", \"XR_VIEWS\", \"WHAT_XR_VIEWS\", \"SHOW_XR_LAUNCHER\"],\n  description:\n    \"Lists all views available on the XR device and optionally sends a launcher catalog to the headset. Use this before XR_OPEN_VIEW.\",\n  examples: [\n    [\n      { name: \"user\", content: { text: \"what can I open in XR?\" } },\n      {\n        name: \"agent\",\n        content: {\n          text: \"Available XR views are registered by the loaded plugins.\",\n          action: \"XR_LIST_VIEWS\",\n        },\n      },\n    ],\n  ],\n\n  validate: async (runtime) => {\n    const svc = getService(runtime);\n    return svc !== null;\n  },\n\n  handler: async (runtime, _message, _state, options, callback) => {\n    const svc = getService(runtime);\n    if (!svc) {\n      const text = \"No XR service.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    // Collect XR view declarations from all registered plugins\n    const xrViews = collectXRViews();\n\n    const connId = firstConnectionId(svc);\n    if (connId && (options?.sendCatalog as boolean | undefined) !== false) {\n      svc.sendViewsCatalog(connId, xrViews);\n    }\n\n    if (xrViews.length === 0) {\n      const text = \"No XR views are currently registered.\";\n      await callback?.({ text });\n      return { success: true, text };\n    }\n\n    const list = xrViews.map((v) => `• ${v.label} (id: ${v.id})`).join(\"\\n\");\n    const text = `Available XR views:\\n${list}\\n\\nSay \"open [view name]\" to launch one.`;\n    await callback?.({ text });\n    return { success: true, text };\n  },\n};\n\n// ── XR_RESIZE_VIEW ─────────────────────────────────────────────────────────\n\nexport const xrResizeViewAction: Action = {\n  name: \"XR_RESIZE_VIEW\",\n  similes: [\"RESIZE_XR_PANEL\", \"XR_MAKE_BIGGER\", \"XR_MAKE_SMALLER\", \"XR_SCALE\"],\n  description:\n    \"Resizes or repositions the active XR view panel. Accepts scale (0.5 = half, 2.0 = double) and distance.\",\n  examples: [\n    [\n      { name: \"user\", content: { text: \"make the panel bigger\" } },\n      {\n        name: \"agent\",\n        content: { text: \"Scaling up the panel.\", action: \"XR_RESIZE_VIEW\" },\n      },\n    ],\n    [\n      { name: \"user\", content: { text: \"make it smaller and move closer\" } },\n      {\n        name: \"agent\",\n        content: {\n          text: \"Resizing and moving panel closer.\",\n          action: \"XR_RESIZE_VIEW\",\n        },\n      },\n    ],\n  ],\n\n  validate: async (runtime) => {\n    const svc = getService(runtime);\n    return svc?.hasActiveConnections() ?? false;\n  },\n\n  handler: async (runtime, message, _state, options, callback) => {\n    const svc = getService(runtime);\n    if (!svc) {\n      const text = \"No XR service.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n    const connId = firstConnectionId(svc);\n    if (!connId) {\n      const text = \"No XR device connected.\";\n      await callback?.({ text });\n      return { success: false, text };\n    }\n\n    const inputText = message.content.text?.toLowerCase() ?? \"\";\n    let scale = (options?.scale as number | undefined) ?? 1.0;\n    let distance = (options?.distance as number | undefined) ?? 1.5;\n\n    if (\n      inputText.includes(\"bigger\") ||\n      inputText.includes(\"larger\") ||\n      inputText.includes(\"bigger\")\n    )\n      scale = 1.5;\n    if (inputText.includes(\"smaller\") || inputText.includes(\"tiny\"))\n      scale = 0.6;\n    if (inputText.includes(\"closer\") || inputText.includes(\"nearer\"))\n      distance = 0.8;\n    if (\n      inputText.includes(\"farther\") ||\n      inputText.includes(\"further\") ||\n      inputText.includes(\"away\")\n    )\n      distance = 2.5;\n    if (inputText.includes(\"fullscreen\") || inputText.includes(\"full screen\")) {\n      svc.resizeView(connId, \"\", { scale: 2.0, fullscreen: true });\n      const text = \"Panel fullscreened.\";\n      await callback?.({ text });\n      return { success: true, text };\n    }\n\n    const viewId = (options?.viewId as string | undefined) ?? \"\";\n    svc.resizeView(connId, viewId, { scale, distance });\n    const text = `Panel resized to ${scale}× at ${distance}m.`;\n    await callback?.({ text });\n    return { success: true, text };\n  },\n};\n\n// ── Helpers ────────────────────────────────────────────────────────────────\n\n/** Extract a likely view id from natural language */\nexport function extractViewId(text: string): string {\n  const lower = text.toLowerCase();\n  for (const view of collectXRViews()) {\n    const terms = [view.id, view.id.replace(/-/g, \" \"), view.label].map(\n      (term) => term.toLowerCase(),\n    );\n    if (terms.some((term) => term && lower.includes(term))) {\n      return view.id;\n    }\n  }\n  const quoted = text.match(/[\"']([^\"']+)[\"']/);\n  if (quoted?.[1]) return quoted[1];\n  return \"\";\n}\n\ntype XRViewSummary = {\n  id: string;\n  label: string;\n  icon?: string;\n  description?: string;\n};\n\n/** Collect all XR-typed views from registered plugins */\nexport function collectXRViews(): XRViewSummary[] {\n  const byId = new Map<string, XRViewSummary>();\n  for (const view of listViews({ developerMode: true, viewType: \"xr\" })) {\n    if (view.viewType !== \"xr\") continue;\n    byId.set(view.id, {\n      id: view.id,\n      label: view.label,\n      icon: view.icon,\n      description: view.description,\n    });\n  }\n  return [...byId.values()].sort(\n    (a, b) => a.label.localeCompare(b.label) || a.id.localeCompare(b.id),\n  );\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAQ1B;AAAA,EACE;AAAA,OAEK;AAIP,SAAS,WAAW,SAAiD;AACnE,SAAO,QAAQ,WAA6B,eAAe,KAAK;AAClE;AAEA,SAAS,kBAAkB,KAAsC;AAC/D,SAAO,IAAI,eAAe,EAAE,CAAC,GAAG,MAAM;AACxC;AAEA,SAAS,eAAuB;AAG9B,QAAM,OACJ,QAAQ,IAAI,gBAAgB,KAAK,KACjC,QAAQ,IAAI,YAAY,KAAK,KAC7B;AACF,SAAO,QAAQ,IAAI,gBAAgB,oBAAoB,IAAI;AAC7D;AAIO,MAAM,mBAA2B;AAAA,EACtC,MAAM;AAAA,EACN,SAAS,CAAC,gBAAgB,iBAAiB,WAAW,WAAW;AAAA,EACjE,aACE;AAAA,EACF,UAAU;AAAA,IACR;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,wBAAwB,EAAE;AAAA,MAC3D;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,gCAAgC,EAAE;AAAA,MACnE;AAAA,QACE,MAAM;AAAA,QACN,SAAS,EAAE,MAAM,6BAA6B,QAAQ,eAAe;AAAA,MACvE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAO,YAA8B;AAC7C,UAAM,MAAM,WAAW,OAAO;AAC9B,WAAO,KAAK,qBAAqB,KAAK;AAAA,EACxC;AAAA,EAEA,SAAS,OACP,SACA,SACA,QACA,SACA,aACG;AACH,UAAM,MAAM,WAAW,OAAO;AAC9B,QAAI,CAAC,KAAK;AACR,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,UAAM,SACH,SAAS,UACV,cAAc,QAAQ,QAAQ,QAAQ,EAAE;AAC1C,QAAI,CAAC,QAAQ;AACX,YAAMA,QACJ;AACF,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,UAAM,SAAS,kBAAkB,GAAG;AACpC,QAAI,CAAC,QAAQ;AACX,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,UAAM,OAAO,aAAa;AAC1B,UAAM,QAAS,SAAS,SAAgC;AACxD,QAAI,SAAS,QAAQ,QAAQ,MAAM,EAAE,OAAO,YAAY,YAAY,CAAC;AACrE,UAAM,OAAO,WAAW,MAAM;AAC9B,UAAM,WAAW,EAAE,KAAK,CAAC;AACzB,WAAO,EAAE,SAAS,MAAM,KAAK;AAAA,EAC/B;AACF;AAIO,MAAM,oBAA4B;AAAA,EACvC,MAAM;AAAA,EACN,SAAS,CAAC,iBAAiB,iBAAiB,YAAY,YAAY;AAAA,EACpE,aAAa;AAAA,EACb,UAAU;AAAA,IACR;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,yBAAyB,EAAE;AAAA,MAC5D;AAAA,QACE,MAAM;AAAA,QACN,SAAS,EAAE,MAAM,yBAAyB,QAAQ,gBAAgB;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAO,YAAY;AAC3B,UAAM,MAAM,WAAW,OAAO;AAC9B,WAAO,KAAK,qBAAqB,KAAK;AAAA,EACxC;AAAA,EAEA,SAAS,OAAO,SAAS,SAAS,QAAQ,SAAS,aAAa;AAC9D,UAAM,MAAM,WAAW,OAAO;AAC9B,QAAI,CAAC,KAAK;AACR,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,UAAM,SACH,SAAS,UACV,cAAc,QAAQ,QAAQ,QAAQ,EAAE;AAC1C,UAAM,SAAS,kBAAkB,GAAG;AACpC,QAAI,CAAC,QAAQ;AACX,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,QAAI;AACJ,QAAI,QAAQ;AACV,UAAI,UAAU,QAAQ,MAAM;AAC5B,aAAO,UAAU,MAAM;AACvB,YAAM,WAAW,EAAE,KAAK,CAAC;AAAA,IAC3B,OAAO;AACL,YAAM,QAAQ,eAAe;AAC7B,iBAAW,QAAQ,OAAO;AACxB,YAAI,UAAU,QAAQ,KAAK,EAAE;AAAA,MAC/B;AACA,aACE,MAAM,SAAS,IAAI,0BAA0B;AAC/C,YAAM,WAAW,EAAE,KAAK,CAAC;AAAA,IAC3B;AACA,WAAO,EAAE,SAAS,MAAM,KAAK;AAAA,EAC/B;AACF;AAIO,MAAM,qBAA6B;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,kBAAkB,YAAY,aAAa;AAAA,EACrD,aACE;AAAA,EACF,UAAU;AAAA,IACR;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,4BAA4B,EAAE;AAAA,MAC/D;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAO,YAAY;AAC3B,UAAM,MAAM,WAAW,OAAO;AAC9B,WAAO,KAAK,qBAAqB,KAAK;AAAA,EACxC;AAAA,EAEA,SAAS,OAAO,SAAS,SAAS,QAAQ,SAAS,aAAa;AAC9D,UAAM,MAAM,WAAW,OAAO;AAC9B,QAAI,CAAC,KAAK;AACR,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AACA,UAAM,SACH,SAAS,UACV,cAAc,QAAQ,QAAQ,QAAQ,EAAE;AAC1C,UAAM,SAAS,kBAAkB,GAAG;AACpC,QAAI,CAAC,UAAU,CAAC,QAAQ;AACtB,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AACA,QAAI,WAAW,QAAQ,MAAM;AAC7B,UAAM,OAAO,eAAe,MAAM;AAClC,UAAM,WAAW,EAAE,KAAK,CAAC;AACzB,WAAO,EAAE,SAAS,MAAM,KAAK;AAAA,EAC/B;AACF;AAIO,MAAM,oBAA4B;AAAA,EACvC,MAAM;AAAA,EACN,SAAS,CAAC,iBAAiB,YAAY,iBAAiB,kBAAkB;AAAA,EAC1E,aACE;AAAA,EACF,UAAU;AAAA,IACR;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,yBAAyB,EAAE;AAAA,MAC5D;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAO,YAAY;AAC3B,UAAM,MAAM,WAAW,OAAO;AAC9B,WAAO,QAAQ;AAAA,EACjB;AAAA,EAEA,SAAS,OAAO,SAAS,UAAU,QAAQ,SAAS,aAAa;AAC/D,UAAM,MAAM,WAAW,OAAO;AAC9B,QAAI,CAAC,KAAK;AACR,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAGA,UAAM,UAAU,eAAe;AAE/B,UAAM,SAAS,kBAAkB,GAAG;AACpC,QAAI,UAAW,SAAS,gBAAwC,OAAO;AACrE,UAAI,iBAAiB,QAAQ,OAAO;AAAA,IACtC;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,MAAM,MAAAA,MAAK;AAAA,IAC/B;AAEA,UAAM,OAAO,QAAQ,IAAI,CAAC,MAAM,UAAK,EAAE,KAAK,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,IAAI;AACvE,UAAM,OAAO;AAAA,EAAwB,IAAI;AAAA;AAAA;AACzC,UAAM,WAAW,EAAE,KAAK,CAAC;AACzB,WAAO,EAAE,SAAS,MAAM,KAAK;AAAA,EAC/B;AACF;AAIO,MAAM,qBAA6B;AAAA,EACxC,MAAM;AAAA,EACN,SAAS,CAAC,mBAAmB,kBAAkB,mBAAmB,UAAU;AAAA,EAC5E,aACE;AAAA,EACF,UAAU;AAAA,IACR;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,wBAAwB,EAAE;AAAA,MAC3D;AAAA,QACE,MAAM;AAAA,QACN,SAAS,EAAE,MAAM,yBAAyB,QAAQ,iBAAiB;AAAA,MACrE;AAAA,IACF;AAAA,IACA;AAAA,MACE,EAAE,MAAM,QAAQ,SAAS,EAAE,MAAM,kCAAkC,EAAE;AAAA,MACrE;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,UAAU,OAAO,YAAY;AAC3B,UAAM,MAAM,WAAW,OAAO;AAC9B,WAAO,KAAK,qBAAqB,KAAK;AAAA,EACxC;AAAA,EAEA,SAAS,OAAO,SAAS,SAAS,QAAQ,SAAS,aAAa;AAC9D,UAAM,MAAM,WAAW,OAAO;AAC9B,QAAI,CAAC,KAAK;AACR,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AACA,UAAM,SAAS,kBAAkB,GAAG;AACpC,QAAI,CAAC,QAAQ;AACX,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,OAAO,MAAAA,MAAK;AAAA,IAChC;AAEA,UAAM,YAAY,QAAQ,QAAQ,MAAM,YAAY,KAAK;AACzD,QAAI,QAAS,SAAS,SAAgC;AACtD,QAAI,WAAY,SAAS,YAAmC;AAE5D,QACE,UAAU,SAAS,QAAQ,KAC3B,UAAU,SAAS,QAAQ,KAC3B,UAAU,SAAS,QAAQ;AAE3B,cAAQ;AACV,QAAI,UAAU,SAAS,SAAS,KAAK,UAAU,SAAS,MAAM;AAC5D,cAAQ;AACV,QAAI,UAAU,SAAS,QAAQ,KAAK,UAAU,SAAS,QAAQ;AAC7D,iBAAW;AACb,QACE,UAAU,SAAS,SAAS,KAC5B,UAAU,SAAS,SAAS,KAC5B,UAAU,SAAS,MAAM;AAEzB,iBAAW;AACb,QAAI,UAAU,SAAS,YAAY,KAAK,UAAU,SAAS,aAAa,GAAG;AACzE,UAAI,WAAW,QAAQ,IAAI,EAAE,OAAO,GAAK,YAAY,KAAK,CAAC;AAC3D,YAAMA,QAAO;AACb,YAAM,WAAW,EAAE,MAAAA,MAAK,CAAC;AACzB,aAAO,EAAE,SAAS,MAAM,MAAAA,MAAK;AAAA,IAC/B;AAEA,UAAM,SAAU,SAAS,UAAiC;AAC1D,QAAI,WAAW,QAAQ,QAAQ,EAAE,OAAO,SAAS,CAAC;AAClD,UAAM,OAAO,oBAAoB,KAAK,WAAQ,QAAQ;AACtD,UAAM,WAAW,EAAE,KAAK,CAAC;AACzB,WAAO,EAAE,SAAS,MAAM,KAAK;AAAA,EAC/B;AACF;AAKO,SAAS,cAAc,MAAsB;AAClD,QAAM,QAAQ,KAAK,YAAY;AAC/B,aAAW,QAAQ,eAAe,GAAG;AACnC,UAAM,QAAQ,CAAC,KAAK,IAAI,KAAK,GAAG,QAAQ,MAAM,GAAG,GAAG,KAAK,KAAK,EAAE;AAAA,MAC9D,CAAC,SAAS,KAAK,YAAY;AAAA,IAC7B;AACA,QAAI,MAAM,KAAK,CAAC,SAAS,QAAQ,MAAM,SAAS,IAAI,CAAC,GAAG;AACtD,aAAO,KAAK;AAAA,IACd;AAAA,EACF;AACA,QAAM,SAAS,KAAK,MAAM,kBAAkB;AAC5C,MAAI,SAAS,CAAC,EAAG,QAAO,OAAO,CAAC;AAChC,SAAO;AACT;AAUO,SAAS,iBAAkC;AAChD,QAAM,OAAO,oBAAI,IAA2B;AAC5C,aAAW,QAAQ,UAAU,EAAE,eAAe,MAAM,UAAU,KAAK,CAAC,GAAG;AACrE,QAAI,KAAK,aAAa,KAAM;AAC5B,SAAK,IAAI,KAAK,IAAI;AAAA,MAChB,IAAI,KAAK;AAAA,MACT,OAAO,KAAK;AAAA,MACZ,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,IACpB,CAAC;AAAA,EACH;AACA,SAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE;AAAA,IACxB,CAAC,GAAG,MAAM,EAAE,MAAM,cAAc,EAAE,KAAK,KAAK,EAAE,GAAG,cAAc,EAAE,EAAE;AAAA,EACrE;AACF;","names":["text"]}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+import type { Plugin } from "@elizaos/core";
+export type * from "./protocol.ts";
+export { AudioPipeline } from "./services/audio-pipeline.ts";
+export { VisionPipeline } from "./services/vision-pipeline.ts";
+export { XR_SERVICE_TYPE, XR_WS_PORT_DEFAULT, XRSessionService, } from "./services/xr-session-service.ts";
+export declare const xrPlugin: Plugin;
+export default xrPlugin;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAiB5C,mBAAmB,eAAe,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,kCAAkC,CAAC;AAE1C,eAAO,MAAM,QAAQ,EAAE,MA0BtB,CAAC;AAEF,eAAe,QAAQ,CAAC"}

package/dist/index.js

@@ -0,0 +1,57 @@
+import { xrQueryVisionAction } from "./actions/xr-query-vision.js";
+import {
+  xrCloseViewAction,
+  xrListViewsAction,
+  xrOpenViewAction,
+  xrResizeViewAction,
+  xrSwitchViewAction
+} from "./actions/xr-view-actions.js";
+import { xrContextProvider } from "./providers/xr-context.js";
+import { xrConnectRoute } from "./routes/xr-connect.js";
+import { xrSimulatorRoute } from "./routes/xr-simulator-route.js";
+import { xrStatusRoute } from "./routes/xr-status.js";
+import { xrViewHostRoute } from "./routes/xr-view-host.js";
+import { xrViewsRoute } from "./routes/xr-views.js";
+import { XRSessionService } from "./services/xr-session-service.js";
+import { AudioPipeline } from "./services/audio-pipeline.js";
+import { VisionPipeline } from "./services/vision-pipeline.js";
+import {
+  XR_SERVICE_TYPE,
+  XR_WS_PORT_DEFAULT,
+  XRSessionService as XRSessionService2
+} from "./services/xr-session-service.js";
+const xrPlugin = {
+  name: "@elizaos/plugin-xr",
+  description: "Streams audio and camera video from XR headsets (Quest 3, XReal) to the agent and delivers voice responses back.",
+  services: [XRSessionService],
+  actions: [
+    xrQueryVisionAction,
+    xrOpenViewAction,
+    xrCloseViewAction,
+    xrSwitchViewAction,
+    xrListViewsAction,
+    xrResizeViewAction
+  ],
+  providers: [xrContextProvider],
+  routes: [
+    xrStatusRoute,
+    xrConnectRoute,
+    xrViewsRoute,
+    xrViewHostRoute,
+    xrSimulatorRoute
+  ],
+  config: {
+    XR_WS_PORT: 31338
+  }
+};
+var index_default = xrPlugin;
+export {
+  AudioPipeline,
+  VisionPipeline,
+  XRSessionService2 as XRSessionService,
+  XR_SERVICE_TYPE,
+  XR_WS_PORT_DEFAULT,
+  index_default as default,
+  xrPlugin
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { Plugin } from \"@elizaos/core\";\nimport { xrQueryVisionAction } from \"./actions/xr-query-vision.js\";\nimport {\n  xrCloseViewAction,\n  xrListViewsAction,\n  xrOpenViewAction,\n  xrResizeViewAction,\n  xrSwitchViewAction,\n} from \"./actions/xr-view-actions.js\";\nimport { xrContextProvider } from \"./providers/xr-context.js\";\nimport { xrConnectRoute } from \"./routes/xr-connect.js\";\nimport { xrSimulatorRoute } from \"./routes/xr-simulator-route.js\";\nimport { xrStatusRoute } from \"./routes/xr-status.js\";\nimport { xrViewHostRoute } from \"./routes/xr-view-host.js\";\nimport { xrViewsRoute } from \"./routes/xr-views.js\";\nimport { XRSessionService } from \"./services/xr-session-service.js\";\n\nexport type * from \"./protocol.js\";\nexport { AudioPipeline } from \"./services/audio-pipeline.js\";\nexport { VisionPipeline } from \"./services/vision-pipeline.js\";\nexport {\n  XR_SERVICE_TYPE,\n  XR_WS_PORT_DEFAULT,\n  XRSessionService,\n} from \"./services/xr-session-service.js\";\n\nexport const xrPlugin: Plugin = {\n  name: \"@elizaos/plugin-xr\",\n  description:\n    \"Streams audio and camera video from XR headsets (Quest 3, XReal) to the agent and delivers voice responses back.\",\n\n  services: [XRSessionService],\n  actions: [\n    xrQueryVisionAction,\n    xrOpenViewAction,\n    xrCloseViewAction,\n    xrSwitchViewAction,\n    xrListViewsAction,\n    xrResizeViewAction,\n  ],\n  providers: [xrContextProvider],\n  routes: [\n    xrStatusRoute,\n    xrConnectRoute,\n    xrViewsRoute,\n    xrViewHostRoute,\n    xrSimulatorRoute,\n  ],\n\n  config: {\n    XR_WS_PORT: 31338,\n  },\n};\n\nexport default xrPlugin;\n"],"mappings":"AACA,SAAS,2BAA2B;AACpC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAC/B,SAAS,wBAAwB;AACjC,SAAS,qBAAqB;AAC9B,SAAS,uBAAuB;AAChC,SAAS,oBAAoB;AAC7B,SAAS,wBAAwB;AAGjC,SAAS,qBAAqB;AAC9B,SAAS,sBAAsB;AAC/B;AAAA,EACE;AAAA,EACA;AAAA,EACA,oBAAAA;AAAA,OACK;AAEA,MAAM,WAAmB;AAAA,EAC9B,MAAM;AAAA,EACN,aACE;AAAA,EAEF,UAAU,CAAC,gBAAgB;AAAA,EAC3B,SAAS;AAAA,IACP;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EACA,WAAW,CAAC,iBAAiB;AAAA,EAC7B,QAAQ;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EAEA,QAAQ;AAAA,IACN,YAAY;AAAA,EACd;AACF;AAEA,IAAO,gBAAQ;","names":["XRSessionService"]}

package/dist/protocol.d.ts

@@ -0,0 +1,124 @@
+export type XRDeviceType = "quest3" | "xreal" | "simulator";
+/** View panel state reported back from the XR device */
+export interface XRViewPanelState {
+    viewId: string;
+    active: boolean;
+    width?: number;
+    height?: number;
+}
+export type XRClientControl = {
+    type: "hello";
+    deviceType: XRDeviceType;
+    sessionId: string;
+} | {
+    type: "ping";
+} | {
+    type: "view_ready";
+    viewId: string;
+} | {
+    type: "view_closed";
+    viewId: string;
+} | {
+    type: "view_event";
+    viewId: string;
+    event: string;
+    payload?: unknown;
+};
+export interface XRAudioHeader {
+    type: "audio";
+    ts: number;
+    sampleRate: number;
+    /** "webm-opus" from MediaRecorder, "pcm-f32" from ScriptProcessor fallback */
+    encoding: "webm-opus" | "pcm-f32";
+}
+export interface XRFrameHeader {
+    type: "frame";
+    ts: number;
+    width: number;
+    height: number;
+    format: "jpeg" | "webp";
+    pose?: {
+        position: {
+            x: number;
+            y: number;
+            z: number;
+        };
+        orientation: {
+            x: number;
+            y: number;
+            z: number;
+            w: number;
+        };
+    };
+}
+export type XRBinaryHeader = XRAudioHeader | XRFrameHeader;
+/** XR panel sizing options */
+export interface XRPanelConfig {
+    /** Panel width relative to default (0.5 = half, 2.0 = double) */
+    scale?: number;
+    /** Follow mode: billboard | fixed | follow */
+    followMode?: "billboard" | "fixed" | "follow";
+    /** Distance from camera in metres */
+    distance?: number;
+    /** Whether to show as full-overlay or floating panel */
+    fullscreen?: boolean;
+}
+export type XRServerControl = {
+    type: "ready";
+    sessionId: string;
+} | {
+    type: "transcript";
+    text: string;
+    final: boolean;
+} | {
+    type: "agent_text";
+    text: string;
+} | {
+    type: "pong";
+}
+/** Open (or bring to front) a view by its registered view id */
+ | {
+    type: "view_open";
+    viewId: string;
+    agentBaseUrl: string;
+    config?: XRPanelConfig;
+}
+/** Close a specific view panel */
+ | {
+    type: "view_close";
+    viewId: string;
+}
+/** Switch the "active" (foreground) view */
+ | {
+    type: "view_switch";
+    viewId: string;
+}
+/** Resize / reposition the active or named panel */
+ | {
+    type: "view_resize";
+    viewId?: string;
+    config: XRPanelConfig;
+}
+/** Send all available views to the device for the launcher */
+ | {
+    type: "views_catalog";
+    views: Array<{
+        id: string;
+        label: string;
+        icon?: string;
+        description?: string;
+    }>;
+};
+export interface XRTTSAudioHeader {
+    type: "tts_audio";
+    sampleRate: number;
+    channels: number;
+    /** encoding of the outbound audio */
+    encoding: "mp3" | "wav" | "pcm-f32";
+}
+export declare function encodeBinaryFrame(header: XRBinaryHeader | XRTTSAudioHeader, payload: Uint8Array | Buffer): Buffer;
+export declare function decodeBinaryFrame(data: Buffer): {
+    header: XRBinaryHeader | XRTTSAudioHeader;
+    payload: Buffer;
+};
+//# sourceMappingURL=protocol.d.ts.map

package/dist/protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,OAAO,GAAG,WAAW,CAAC;AAI5D,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9D;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAI7E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,QAAQ,EAAE,WAAW,GAAG,SAAS,CAAC;CACnC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE;QACL,QAAQ,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9C,WAAW,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7D,CAAC;CACH;AAED,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,aAAa,CAAC;AAI3D,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,UAAU,CAAC,EAAE,WAAW,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC9C,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE;AAElB,gEAAgE;GAC9D;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,aAAa,CAAC;CACxB;AACH,kCAAkC;GAChC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACxC,4CAA4C;GAC1C;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACzC,oDAAoD;GAClD;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE;AACjE,8DAA8D;GAC5D;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACJ,CAAC;AAIN,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,QAAQ,EAAE,KAAK,GAAG,KAAK,GAAG,SAAS,CAAC;CACrC;AAID,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,cAAc,GAAG,gBAAgB,EACzC,OAAO,EAAE,UAAU,GAAG,MAAM,GAC3B,MAAM,CAKR;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG;IAC/C,MAAM,EAAE,cAAc,GAAG,gBAAgB,CAAC;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA"}

package/dist/protocol.js

@@ -0,0 +1,18 @@
+function encodeBinaryFrame(header, payload) {
+  const headerJson = Buffer.from(JSON.stringify(header), "utf8");
+  const lenBuf = Buffer.allocUnsafe(4);
+  lenBuf.writeUInt32BE(headerJson.length, 0);
+  return Buffer.concat([lenBuf, headerJson, payload]);
+}
+function decodeBinaryFrame(data) {
+  const headerLen = data.readUInt32BE(0);
+  const headerJson = data.subarray(4, 4 + headerLen).toString("utf8");
+  const header = JSON.parse(headerJson);
+  const payload = data.subarray(4 + headerLen);
+  return { header, payload };
+}
+export {
+  decodeBinaryFrame,
+  encodeBinaryFrame
+};
+//# sourceMappingURL=protocol.js.map