@elizaos/plugin-xr 2.0.3-beta.6 → 2.0.3-beta.7

package/src/actions/xr-view-actions.ts

@@ -1,386 +0,0 @@
-import { listViews } from "@elizaos/agent/api/views-registry";
-import type {
-  Action,
-  HandlerCallback,
-  IAgentRuntime,
-  Memory,
-  State,
-} from "@elizaos/core";
-import {
-  XR_SERVICE_TYPE,
-  type XRSessionService,
-} from "../services/xr-session-service.ts";
-
-// ── Helpers ────────────────────────────────────────────────────────────────
-
-function getService(runtime: IAgentRuntime): XRSessionService | null {
-  return runtime.getService<XRSessionService>(XR_SERVICE_TYPE) ?? null;
-}
-
-function firstConnectionId(svc: XRSessionService): string | null {
-  return svc.getConnections()[0]?.id ?? null;
-}
-
-function agentBaseUrl(): string {
-  // The API port is orchestrator-assigned (never hardcoded) — read it from the
-  // environment, not a non-existent `runtime.port` field. Falls back to 31337.
-  const port =
-    process.env.ELIZA_API_PORT?.trim() ||
-    process.env.ELIZA_PORT?.trim() ||
-    "31337";
-  return process.env.XR_AGENT_URL ?? `http://localhost:${port}`;
-}
-
-// ── XR_OPEN_VIEW ───────────────────────────────────────────────────────────
-
-export const xrOpenViewAction: Action = {
-  name: "XR_OPEN_VIEW",
-  similes: ["OPEN_XR_VIEW", "SHOW_XR_PANEL", "XR_SHOW", "XR_LAUNCH"],
-  description:
-    "Opens a view panel on the connected XR headset by view id. Use XR_LIST_VIEWS first to discover available view ids.",
-  examples: [
-    [
-      { name: "user", content: { text: "open the wallet in XR" } },
-      {
-        name: "agent",
-        content: {
-          text: "Opening wallet view on your headset.",
-          action: "XR_OPEN_VIEW",
-        },
-      },
-    ],
-    [
-      { name: "user", content: { text: "show training dashboard in XR" } },
-      {
-        name: "agent",
-        content: { text: "Launching training panel.", action: "XR_OPEN_VIEW" },
-      },
-    ],
-  ],
-
-  validate: async (runtime): Promise<boolean> => {
-    const svc = getService(runtime);
-    return svc?.hasActiveConnections() ?? false;
-  },
-
-  handler: async (
-    runtime: IAgentRuntime,
-    message: Memory,
-    _state: State | undefined,
-    options: Record<string, unknown> | undefined,
-    callback?: HandlerCallback,
-  ) => {
-    const svc = getService(runtime);
-    if (!svc) {
-      const text = "No XR session service available.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    const viewId =
-      (options?.viewId as string | undefined) ??
-      extractViewId(message.content.text ?? "");
-    if (!viewId) {
-      const text =
-        "Please specify which view to open. Try XR_LIST_VIEWS to see available views.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    const connId = firstConnectionId(svc);
-    if (!connId) {
-      const text = "No XR device is connected.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    const base = agentBaseUrl();
-    const scale = (options?.scale as number | undefined) ?? 1.0;
-    svc.openView(connId, viewId, base, { scale, followMode: "billboard" });
-    const text = `Opening ${viewId} view on your headset.`;
-    await callback?.({ text });
-    return { success: true, text };
-  },
-};
-
-// ── XR_CLOSE_VIEW ──────────────────────────────────────────────────────────
-
-export const xrCloseViewAction: Action = {
-  name: "XR_CLOSE_VIEW",
-  similes: ["CLOSE_XR_VIEW", "HIDE_XR_PANEL", "XR_CLOSE", "XR_DISMISS"],
-  description: "Closes a specific view panel on the connected XR headset.",
-  examples: [
-    [
-      { name: "user", content: { text: "close the wallet panel" } },
-      {
-        name: "agent",
-        content: { text: "Closing wallet panel.", action: "XR_CLOSE_VIEW" },
-      },
-    ],
-  ],
-
-  validate: async (runtime) => {
-    const svc = getService(runtime);
-    return svc?.hasActiveConnections() ?? false;
-  },
-
-  handler: async (runtime, message, _state, options, callback) => {
-    const svc = getService(runtime);
-    if (!svc) {
-      const text = "No XR service.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    const viewId =
-      (options?.viewId as string | undefined) ??
-      extractViewId(message.content.text ?? "");
-    const connId = firstConnectionId(svc);
-    if (!connId) {
-      const text = "No XR device connected.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    let text: string;
-    if (viewId) {
-      svc.closeView(connId, viewId);
-      text = `Closed ${viewId}.`;
-      await callback?.({ text });
-    } else {
-      const views = collectXRViews();
-      for (const view of views) {
-        svc.closeView(connId, view.id);
-      }
-      text =
-        views.length > 0 ? "Closed all XR panels." : "No XR panels to close.";
-      await callback?.({ text });
-    }
-    return { success: true, text };
-  },
-};
-
-// ── XR_SWITCH_VIEW ─────────────────────────────────────────────────────────
-
-export const xrSwitchViewAction: Action = {
-  name: "XR_SWITCH_VIEW",
-  similes: ["SWITCH_XR_VIEW", "XR_GO_TO", "XR_NAVIGATE"],
-  description:
-    "Switches the active (foreground) view on the XR headset without closing others.",
-  examples: [
-    [
-      { name: "user", content: { text: "switch to companion in XR" } },
-      {
-        name: "agent",
-        content: {
-          text: "Switching to companion view.",
-          action: "XR_SWITCH_VIEW",
-        },
-      },
-    ],
-  ],
-
-  validate: async (runtime) => {
-    const svc = getService(runtime);
-    return svc?.hasActiveConnections() ?? false;
-  },
-
-  handler: async (runtime, message, _state, options, callback) => {
-    const svc = getService(runtime);
-    if (!svc) {
-      const text = "No XR service.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-    const viewId =
-      (options?.viewId as string | undefined) ??
-      extractViewId(message.content.text ?? "");
-    const connId = firstConnectionId(svc);
-    if (!connId || !viewId) {
-      const text = "Specify a view id.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-    svc.switchView(connId, viewId);
-    const text = `Switched to ${viewId}.`;
-    await callback?.({ text });
-    return { success: true, text };
-  },
-};
-
-// ── XR_LIST_VIEWS ──────────────────────────────────────────────────────────
-
-export const xrListViewsAction: Action = {
-  name: "XR_LIST_VIEWS",
-  similes: ["LIST_XR_VIEWS", "XR_VIEWS", "WHAT_XR_VIEWS", "SHOW_XR_LAUNCHER"],
-  description:
-    "Lists all views available on the XR device and optionally sends a launcher catalog to the headset. Use this before XR_OPEN_VIEW.",
-  examples: [
-    [
-      { name: "user", content: { text: "what can I open in XR?" } },
-      {
-        name: "agent",
-        content: {
-          text: "Available XR views are registered by the loaded plugins.",
-          action: "XR_LIST_VIEWS",
-        },
-      },
-    ],
-  ],
-
-  validate: async (runtime) => {
-    const svc = getService(runtime);
-    return svc !== null;
-  },
-
-  handler: async (runtime, _message, _state, options, callback) => {
-    const svc = getService(runtime);
-    if (!svc) {
-      const text = "No XR service.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    // Collect XR view declarations from all registered plugins
-    const xrViews = collectXRViews();
-
-    const connId = firstConnectionId(svc);
-    if (connId && (options?.sendCatalog as boolean | undefined) !== false) {
-      svc.sendViewsCatalog(connId, xrViews);
-    }
-
-    if (xrViews.length === 0) {
-      const text = "No XR views are currently registered.";
-      await callback?.({ text });
-      return { success: true, text };
-    }
-
-    const list = xrViews.map((v) => `• ${v.label} (id: ${v.id})`).join("\n");
-    const text = `Available XR views:\n${list}\n\nSay "open [view name]" to launch one.`;
-    await callback?.({ text });
-    return { success: true, text };
-  },
-};
-
-// ── XR_RESIZE_VIEW ─────────────────────────────────────────────────────────
-
-export const xrResizeViewAction: Action = {
-  name: "XR_RESIZE_VIEW",
-  similes: ["RESIZE_XR_PANEL", "XR_MAKE_BIGGER", "XR_MAKE_SMALLER", "XR_SCALE"],
-  description:
-    "Resizes or repositions the active XR view panel. Accepts scale (0.5 = half, 2.0 = double) and distance.",
-  examples: [
-    [
-      { name: "user", content: { text: "make the panel bigger" } },
-      {
-        name: "agent",
-        content: { text: "Scaling up the panel.", action: "XR_RESIZE_VIEW" },
-      },
-    ],
-    [
-      { name: "user", content: { text: "make it smaller and move closer" } },
-      {
-        name: "agent",
-        content: {
-          text: "Resizing and moving panel closer.",
-          action: "XR_RESIZE_VIEW",
-        },
-      },
-    ],
-  ],
-
-  validate: async (runtime) => {
-    const svc = getService(runtime);
-    return svc?.hasActiveConnections() ?? false;
-  },
-
-  handler: async (runtime, message, _state, options, callback) => {
-    const svc = getService(runtime);
-    if (!svc) {
-      const text = "No XR service.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-    const connId = firstConnectionId(svc);
-    if (!connId) {
-      const text = "No XR device connected.";
-      await callback?.({ text });
-      return { success: false, text };
-    }
-
-    const inputText = message.content.text?.toLowerCase() ?? "";
-    let scale = (options?.scale as number | undefined) ?? 1.0;
-    let distance = (options?.distance as number | undefined) ?? 1.5;
-
-    if (
-      inputText.includes("bigger") ||
-      inputText.includes("larger") ||
-      inputText.includes("bigger")
-    )
-      scale = 1.5;
-    if (inputText.includes("smaller") || inputText.includes("tiny"))
-      scale = 0.6;
-    if (inputText.includes("closer") || inputText.includes("nearer"))
-      distance = 0.8;
-    if (
-      inputText.includes("farther") ||
-      inputText.includes("further") ||
-      inputText.includes("away")
-    )
-      distance = 2.5;
-    if (inputText.includes("fullscreen") || inputText.includes("full screen")) {
-      svc.resizeView(connId, "", { scale: 2.0, fullscreen: true });
-      const text = "Panel fullscreened.";
-      await callback?.({ text });
-      return { success: true, text };
-    }
-
-    const viewId = (options?.viewId as string | undefined) ?? "";
-    svc.resizeView(connId, viewId, { scale, distance });
-    const text = `Panel resized to ${scale}× at ${distance}m.`;
-    await callback?.({ text });
-    return { success: true, text };
-  },
-};
-
-// ── Helpers ────────────────────────────────────────────────────────────────
-
-/** Extract a likely view id from natural language */
-export function extractViewId(text: string): string {
-  const lower = text.toLowerCase();
-  for (const view of collectXRViews()) {
-    const terms = [view.id, view.id.replace(/-/g, " "), view.label].map(
-      (term) => term.toLowerCase(),
-    );
-    if (terms.some((term) => term && lower.includes(term))) {
-      return view.id;
-    }
-  }
-  const quoted = text.match(/["']([^"']+)["']/);
-  if (quoted?.[1]) return quoted[1];
-  return "";
-}
-
-type XRViewSummary = {
-  id: string;
-  label: string;
-  icon?: string;
-  description?: string;
-};
-
-/** Collect all XR-typed views from registered plugins */
-export function collectXRViews(): XRViewSummary[] {
-  const byId = new Map<string, XRViewSummary>();
-  for (const view of listViews({ developerMode: true, viewType: "xr" })) {
-    if (view.viewType !== "xr") continue;
-    byId.set(view.id, {
-      id: view.id,
-      label: view.label,
-      icon: view.icon,
-      description: view.description,
-    });
-  }
-  return [...byId.values()].sort(
-    (a, b) => a.label.localeCompare(b.label) || a.id.localeCompare(b.id),
-  );
-}

package/src/index.ts

@@ -1,55 +0,0 @@
-import type { Plugin } from "@elizaos/core";
-import { xrQueryVisionAction } from "./actions/xr-query-vision.ts";
-import {
-  xrCloseViewAction,
-  xrListViewsAction,
-  xrOpenViewAction,
-  xrResizeViewAction,
-  xrSwitchViewAction,
-} from "./actions/xr-view-actions.ts";
-import { xrContextProvider } from "./providers/xr-context.ts";
-import { xrConnectRoute } from "./routes/xr-connect.ts";
-import { xrSimulatorRoute } from "./routes/xr-simulator-route.ts";
-import { xrStatusRoute } from "./routes/xr-status.ts";
-import { xrViewHostRoute } from "./routes/xr-view-host.ts";
-import { xrViewsRoute } from "./routes/xr-views.ts";
-import { XRSessionService } from "./services/xr-session-service.ts";
-
-export type * from "./protocol.ts";
-export { AudioPipeline } from "./services/audio-pipeline.ts";
-export { VisionPipeline } from "./services/vision-pipeline.ts";
-export {
-  XR_SERVICE_TYPE,
-  XR_WS_PORT_DEFAULT,
-  XRSessionService,
-} from "./services/xr-session-service.ts";
-
-export const xrPlugin: Plugin = {
-  name: "@elizaos/plugin-xr",
-  description:
-    "Streams audio and camera video from XR headsets (Quest 3, XReal) to the agent and delivers voice responses back.",
-
-  services: [XRSessionService],
-  actions: [
-    xrQueryVisionAction,
-    xrOpenViewAction,
-    xrCloseViewAction,
-    xrSwitchViewAction,
-    xrListViewsAction,
-    xrResizeViewAction,
-  ],
-  providers: [xrContextProvider],
-  routes: [
-    xrStatusRoute,
-    xrConnectRoute,
-    xrViewsRoute,
-    xrViewHostRoute,
-    xrSimulatorRoute,
-  ],
-
-  config: {
-    XR_WS_PORT: 31338,
-  },
-};
-
-export default xrPlugin;

package/src/protocol.ts

@@ -1,126 +0,0 @@
-// Binary frame layout:
-//   bytes 0–3  : big-endian uint32 — JSON header length
-//   bytes 4–N  : UTF-8 JSON header
-//   bytes N+1… : raw binary payload (audio PCM/Opus, JPEG, etc.)
-//
-// Text frames are JSON control messages (no binary payload).
-
-export type XRDeviceType = "quest3" | "xreal" | "simulator";
-
-// ── Client → Server (text frames) ──────────────────────────────────────────
-
-/** View panel state reported back from the XR device */
-export interface XRViewPanelState {
-  viewId: string;
-  active: boolean;
-  width?: number;
-  height?: number;
-}
-
-export type XRClientControl =
-  | { type: "hello"; deviceType: XRDeviceType; sessionId: string }
-  | { type: "ping" }
-  | { type: "view_ready"; viewId: string }
-  | { type: "view_closed"; viewId: string }
-  | { type: "view_event"; viewId: string; event: string; payload?: unknown };
-
-// ── Client → Server (binary frames) ────────────────────────────────────────
-
-export interface XRAudioHeader {
-  type: "audio";
-  ts: number;
-  sampleRate: number;
-  /** "webm-opus" from MediaRecorder, "pcm-f32" from ScriptProcessor fallback */
-  encoding: "webm-opus" | "pcm-f32";
-}
-
-export interface XRFrameHeader {
-  type: "frame";
-  ts: number;
-  width: number;
-  height: number;
-  format: "jpeg" | "webp";
-  pose?: {
-    position: { x: number; y: number; z: number };
-    orientation: { x: number; y: number; z: number; w: number };
-  };
-}
-
-export type XRBinaryHeader = XRAudioHeader | XRFrameHeader;
-
-// ── Server → Client (text frames) ──────────────────────────────────────────
-
-/** XR panel sizing options */
-export interface XRPanelConfig {
-  /** Panel width relative to default (0.5 = half, 2.0 = double) */
-  scale?: number;
-  /** Follow mode: billboard | fixed | follow */
-  followMode?: "billboard" | "fixed" | "follow";
-  /** Distance from camera in metres */
-  distance?: number;
-  /** Whether to show as full-overlay or floating panel */
-  fullscreen?: boolean;
-}
-
-export type XRServerControl =
-  | { type: "ready"; sessionId: string }
-  | { type: "transcript"; text: string; final: boolean }
-  | { type: "agent_text"; text: string }
-  | { type: "pong" }
-  // ── View commands ────────────────────────────────────────────────────────
-  /** Open (or bring to front) a view by its registered view id */
-  | {
-      type: "view_open";
-      viewId: string;
-      agentBaseUrl: string;
-      config?: XRPanelConfig;
-    }
-  /** Close a specific view panel */
-  | { type: "view_close"; viewId: string }
-  /** Switch the "active" (foreground) view */
-  | { type: "view_switch"; viewId: string }
-  /** Resize / reposition the active or named panel */
-  | { type: "view_resize"; viewId?: string; config: XRPanelConfig }
-  /** Send all available views to the device for the launcher */
-  | {
-      type: "views_catalog";
-      views: Array<{
-        id: string;
-        label: string;
-        icon?: string;
-        description?: string;
-      }>;
-    };
-
-// ── Server → Client (binary frames) ────────────────────────────────────────
-
-export interface XRTTSAudioHeader {
-  type: "tts_audio";
-  sampleRate: number;
-  channels: number;
-  /** encoding of the outbound audio */
-  encoding: "mp3" | "wav" | "pcm-f32";
-}
-
-// ── Framing helpers ─────────────────────────────────────────────────────────
-
-export function encodeBinaryFrame(
-  header: XRBinaryHeader | XRTTSAudioHeader,
-  payload: Uint8Array | Buffer,
-): Buffer {
-  const headerJson = Buffer.from(JSON.stringify(header), "utf8");
-  const lenBuf = Buffer.allocUnsafe(4);
-  lenBuf.writeUInt32BE(headerJson.length, 0);
-  return Buffer.concat([lenBuf, headerJson, payload]);
-}
-
-export function decodeBinaryFrame(data: Buffer): {
-  header: XRBinaryHeader | XRTTSAudioHeader;
-  payload: Buffer;
-} {
-  const headerLen = data.readUInt32BE(0);
-  const headerJson = data.subarray(4, 4 + headerLen).toString("utf8");
-  const header = JSON.parse(headerJson) as XRBinaryHeader | XRTTSAudioHeader;
-  const payload = data.subarray(4 + headerLen);
-  return { header, payload };
-}

package/src/providers/xr-context.ts

@@ -1,49 +0,0 @@
-import type {
-  IAgentRuntime,
-  Memory,
-  Provider,
-  ProviderResult,
-  State,
-} from "@elizaos/core";
-import {
-  XR_SERVICE_TYPE,
-  type XRSessionService,
-} from "../services/xr-session-service.ts";
-
-export const xrContextProvider: Provider = {
-  name: "XR_SESSION",
-  description: "Provides context about connected XR headsets (Quest 3, XReal)",
-
-  get: async (
-    runtime: IAgentRuntime,
-    _message: Memory,
-    _state: State,
-  ): Promise<ProviderResult> => {
-    const svc = runtime.getService<XRSessionService>(XR_SERVICE_TYPE);
-    if (!svc?.hasActiveConnections()) return { text: "" };
-
-    const conns = svc.getConnections();
-    const deviceList = conns
-      .map((c) => {
-        const hasFrame = svc.getVisionPipeline().hasRecentFrame(c.id);
-        return `${c.deviceType}${hasFrame ? " (camera active)" : ""}`;
-      })
-      .join(", ");
-
-    const text = [
-      `[XR devices connected: ${deviceList}]`,
-      `[Audio streaming active — the user is speaking to you via their headset microphone.]`,
-      `[Your text responses will be spoken aloud through the headset via TTS.]`,
-      `[Use XR_QUERY_VISION to describe what the user's camera sees.]`,
-    ].join("\n");
-
-    return {
-      text,
-      values: {
-        xrConnected: true,
-        xrDevices: conns.map((c) => c.deviceType),
-        xrConnectionCount: conns.length,
-      },
-    };
-  },
-};

package/src/routes/xr-views.ts

@@ -1,43 +0,0 @@
-import { listViews } from "@elizaos/agent/api/views-registry";
-import type { Route } from "@elizaos/core";
-import {
-  XR_SERVICE_TYPE,
-  type XRSessionService,
-} from "../services/xr-session-service.ts";
-
-/**
- * GET /api/xr/views
- * Returns all XR-typed views from registered plugins.
- * Used by app-xr to populate the view launcher.
- */
-export const xrViewsRoute: Route = {
-  type: "GET",
-  path: "/xr/views",
-  description: "Lists all XR-capable views from registered plugins",
-  routeHandler: async (ctx) => {
-    const views = listViews({ developerMode: true, viewType: "xr" })
-      .filter((v) => v.viewType === "xr")
-      .map((v) => ({
-        id: v.id,
-        label: v.label,
-        icon: v.icon,
-        description: v.description,
-        tags: v.tags,
-        xrOptions: v.xrOptions,
-        path: v.path,
-        pluginName: v.pluginName,
-        available: v.available,
-      }));
-
-    const connections =
-      ctx.runtime
-        .getService<XRSessionService>(XR_SERVICE_TYPE)
-        ?.getConnections()
-        .map((c) => ({ id: c.id, deviceType: c.deviceType })) ?? [];
-
-    return {
-      status: 200,
-      body: { views, connections, count: views.length },
-    };
-  },
-};