@elizaos/plugin-xr 2.0.3-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/AGENTS.md +151 -0
  2. package/CLAUDE.md +151 -0
  3. package/LICENSE +21 -0
  4. package/README.md +106 -0
  5. package/package.json +57 -0
  6. package/simulator/bun.lock +159 -0
  7. package/simulator/package.json +28 -0
  8. package/simulator/src/emulator.ts +174 -0
  9. package/simulator/src/mock-agent.ts +233 -0
  10. package/simulator/src/node.ts +9 -0
  11. package/simulator/src/playwright-fixture.ts +169 -0
  12. package/simulator/src/types.ts +51 -0
  13. package/simulator/tsconfig.json +13 -0
  14. package/simulator/vite.config.ts +25 -0
  15. package/src/__tests__/audio-pipeline.test.ts +129 -0
  16. package/src/__tests__/protocol.test.ts +53 -0
  17. package/src/__tests__/routes-e2e.test.ts +276 -0
  18. package/src/__tests__/vision-pipeline.test.ts +73 -0
  19. package/src/__tests__/xr-bundle-coverage.test.ts +303 -0
  20. package/src/__tests__/xr-feature-parity.test.ts +524 -0
  21. package/src/__tests__/xr-functional-parity.test.ts +522 -0
  22. package/src/__tests__/xr-view-host-http.test.ts +239 -0
  23. package/src/__tests__/xr-view-host.test.ts +174 -0
  24. package/src/actions/xr-query-vision.ts +64 -0
  25. package/src/actions/xr-view-actions.ts +386 -0
  26. package/src/index.ts +55 -0
  27. package/src/protocol.ts +126 -0
  28. package/src/providers/xr-context.ts +49 -0
  29. package/src/routes/xr-connect.ts +89 -0
  30. package/src/routes/xr-simulator-route.ts +37 -0
  31. package/src/routes/xr-status.ts +36 -0
  32. package/src/routes/xr-view-host.ts +359 -0
  33. package/src/routes/xr-views.ts +43 -0
  34. package/src/services/audio-pipeline.ts +120 -0
  35. package/src/services/vision-pipeline.ts +57 -0
  36. package/src/services/xr-session-service.ts +388 -0
  37. package/tsconfig.build.json +9 -0
  38. package/tsconfig.json +30 -0
  39. package/vitest.config.ts +21 -0
package/src/routes/xr-connect.ts ADDED
@@ -0,0 +1,89 @@
1
+ import { networkInterfaces } from "node:os";
2
+ import type { Route } from "@elizaos/core";
3
+
4
+ function getLocalIp(): string {
5
+ const nets = networkInterfaces();
6
+ for (const iface of Object.values(nets)) {
7
+ for (const net of iface ?? []) {
8
+ if (!net.internal && net.family === "IPv4") return net.address;
9
+ }
10
+ }
11
+ return "127.0.0.1";
12
+ }
13
+
14
+ function getAppUrl(): string {
15
+ // Set by the connect script when a tunnel is active
16
+ if (process.env.XR_APP_URL) return process.env.XR_APP_URL;
17
+ const port = process.env.VITE_PORT ?? "5173";
18
+ return `http://${getLocalIp()}:${port}`;
19
+ }
20
+
21
+ function htmlPage(appUrl: string): string {
22
+ const encoded = encodeURIComponent(appUrl);
23
+ return `<!DOCTYPE html>
24
+ <html lang="en">
25
+ <head>
26
+ <meta charset="utf-8">
27
+ <meta name="viewport" content="width=device-width, initial-scale=1">
28
+ <title>Connect XR Headset</title>
29
+ <style>
30
+ * { box-sizing: border-box; margin: 0; padding: 0 }
31
+ body { font-family: system-ui, sans-serif; background: #0f0f0f; color: #e5e5e5;
32
+ min-height: 100vh; display: flex; align-items: center; justify-content: center; }
33
+ .card { background: #1a1a1a; border: 1px solid #2a2a2a; border-radius: 16px;
34
+ padding: 40px; max-width: 480px; width: 100%; text-align: center; }
35
+ h1 { font-size: 1.5rem; margin-bottom: 8px; }
36
+ .sub { color: #888; font-size: 0.9rem; margin-bottom: 28px; }
37
+ #qrcanvas { border-radius: 12px; background: white; padding: 12px;
38
+ display: block; margin: 0 auto 24px; }
39
+ .url { background: #111; border: 1px solid #333; border-radius: 8px;
40
+ padding: 10px 16px; font-family: monospace; font-size: 0.85rem;
41
+ word-break: break-all; margin-bottom: 24px; }
42
+ .steps { text-align: left; font-size: 0.85rem; color: #aaa; line-height: 1.7; }
43
+ .steps li { margin-bottom: 4px; }
44
+ .warn { margin-top: 20px; font-size: 0.78rem; color: #f59e0b; }
45
+ </style>
46
+ </head>
47
+ <body>
48
+ <div class="card">
49
+ <h1>Connect XR Headset</h1>
50
+ <p class="sub">Scan to open the XR app on your device</p>
51
+ <canvas id="qrcanvas"></canvas>
52
+ <div class="url">${appUrl}</div>
53
+ <ol class="steps">
54
+ <li>Put on your headset and open the browser</li>
55
+ <li>Scan the QR code or type the URL above</li>
56
+ <li>Allow microphone and camera access when prompted</li>
57
+ <li>The agent will connect automatically</li>
58
+ </ol>
59
+ ${appUrl.startsWith("http://") ? `<p class="warn">⚠ HTTP URL — WebXR requires HTTPS on device.<br>Run <code>bun run connect</code> for an HTTPS tunnel.</p>` : ""}
60
+ </div>
61
+ <script src="https://cdn.jsdelivr.net/npm/qrcode@1.5.4/build/qrcode.min.js"></script>
62
+ <script>
63
+ var url = decodeURIComponent("${encoded}");
64
+ if (typeof QRCode !== "undefined") {
65
+ QRCode.toCanvas(document.getElementById("qrcanvas"), url, { width: 220, margin: 2 }, function(err) {
66
+ if (err) document.getElementById("qrcanvas").style.display = "none";
67
+ });
68
+ } else {
69
+ document.getElementById("qrcanvas").style.display = "none";
70
+ }
71
+ </script>
72
+ </body>
73
+ </html>`;
74
+ }
75
+
76
+ export const xrConnectRoute: Route = {
77
+ type: "GET",
78
+ path: "/xr/connect",
79
+ description:
80
+ "Returns an HTML page with a QR code to connect an XR headset. Set XR_APP_URL env var (or run `bun run connect` in plugins/plugin-facewear/app-xr) to show the correct public URL.",
81
+ routeHandler: async (_ctx) => {
82
+ const url = getAppUrl();
83
+ return {
84
+ status: 200,
85
+ headers: { "Content-Type": "text/html; charset=utf-8" },
86
+ body: htmlPage(url),
87
+ };
88
+ },
89
+ };
package/src/routes/xr-simulator-route.ts ADDED
@@ -0,0 +1,37 @@
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import { dirname, resolve } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import type { Route } from "@elizaos/core";
5
+
6
+ const __dirname = dirname(fileURLToPath(import.meta.url));
7
+ const EMULATOR_BUNDLE = resolve(__dirname, "../../simulator/dist/emulator.js");
8
+
9
+ /**
10
+ * Serves the built WebXR emulator bundle at GET /api/xr/simulator.js
11
+ * Only active when the bundle exists (i.e., after `bun run simulator:build`).
12
+ * Used by Playwright tests that prefer HTTP-served scripts over filesystem paths.
13
+ */
14
+ export const xrSimulatorRoute: Route = {
15
+ type: "GET",
16
+ path: "/xr/simulator.js",
17
+ description:
18
+ "Serves the XR device emulator bundle (IWER + camera injection) for Playwright testing. Build first: cd plugins/plugin-xr/simulator && bun run build",
19
+ routeHandler: async (_ctx) => {
20
+ if (!existsSync(EMULATOR_BUNDLE)) {
21
+ return {
22
+ status: 404,
23
+ body: {
24
+ error: "Emulator bundle not built",
25
+ hint: "Run: cd eliza/plugins/plugin-xr/simulator && bun run build",
26
+ },
27
+ };
28
+ }
29
+
30
+ const js = readFileSync(EMULATOR_BUNDLE, "utf8");
31
+ return {
32
+ status: 200,
33
+ headers: { "Content-Type": "application/javascript; charset=utf-8" },
34
+ body: js,
35
+ };
36
+ },
37
+ };
package/src/routes/xr-status.ts ADDED
@@ -0,0 +1,36 @@
1
+ import type { Route } from "@elizaos/core";
2
+ import {
3
+ XR_SERVICE_TYPE,
4
+ type XRSessionService,
5
+ } from "../services/xr-session-service.ts";
6
+
7
+ export const xrStatusRoute: Route = {
8
+ type: "GET",
9
+ path: "/xr/status",
10
+ description: "Returns the list of connected XR devices and session state",
11
+ routeHandler: async (ctx) => {
12
+ const svc = ctx.runtime.getService<XRSessionService>(XR_SERVICE_TYPE);
13
+
14
+ if (!svc) {
15
+ return {
16
+ status: 503,
17
+ body: { error: "XR service not running" },
18
+ };
19
+ }
20
+
21
+ const conns = svc.getConnections().map((c) => ({
22
+ id: c.id,
23
+ deviceType: c.deviceType,
24
+ connectedAt: c.connectedAt.toISOString(),
25
+ hasRecentFrame: svc.getVisionPipeline().hasRecentFrame(c.id),
26
+ }));
27
+
28
+ return {
29
+ status: 200,
30
+ body: {
31
+ connected: conns.length > 0,
32
+ connections: conns,
33
+ },
34
+ };
35
+ },
36
+ };
package/src/routes/xr-view-host.ts ADDED
@@ -0,0 +1,359 @@
1
+ import type { Route } from "@elizaos/core";
2
+
3
+ /**
4
+ * GET /api/xr/view-host/:id
5
+ *
6
+ * Serves a self-contained HTML page that loads and renders a registered
7
+ * elizaOS view bundle inside an XR-optimised shell. App-xr opens this URL
8
+ * in an iframe that is overlaid on the WebXR scene.
9
+ *
10
+ * The host page:
11
+ * 1. Loads React + ReactDOM from the CDN (same versions as the view bundles).
12
+ * 2. Dynamically imports the view bundle from the agent's /api/views/:id/bundle.js.
13
+ * 3. Mounts the view component inside an XR-friendly dark-theme container.
14
+ * 4. Provides a minimal elizaOS context (agentBaseUrl, viewId, postMessage bridge).
15
+ * 5. Routes voice transcript text to the focused form input.
16
+ *
17
+ * Inter-frame communication (postMessage):
18
+ * Parent → host: { type:"xr:transcript", text:"..." } — fill focused input
19
+ * { type:"xr:focus-next" } — tab to next field
20
+ * Host → parent: { type:"xr:view-ready", viewId:"..." }
21
+ * { type:"xr:navigate", viewId:"..." }
22
+ * { type:"xr:close" }
23
+ */
24
+ export const xrViewHostRoute: Route = {
25
+ type: "GET",
26
+ path: "/xr/view-host/:id",
27
+ description:
28
+ "Serves a self-contained XR-friendly HTML host page for a registered view",
29
+ routeHandler: async (ctx) => {
30
+ const viewId = (ctx.params as Record<string, string>)?.id ?? "";
31
+ if (!viewId) {
32
+ return { status: 400, body: { error: "Missing view id" } };
33
+ }
34
+
35
+ // Resolve the agent origin so the page can load the bundle
36
+ const agentPort = (ctx.runtime as { port?: number }).port ?? 31337;
37
+ const agentOrigin =
38
+ process.env.XR_AGENT_URL ?? `http://localhost:${agentPort}`;
39
+ const bundleUrl = `${agentOrigin}/api/views/${viewId}/bundle.js`;
40
+ const viewsApiUrl = `${agentOrigin}/api/views`;
41
+
42
+ const html = buildHostPage(viewId, bundleUrl, viewsApiUrl, agentOrigin);
43
+
44
+ return {
45
+ status: 200,
46
+ headers: {
47
+ "Content-Type": "text/html; charset=utf-8",
48
+ // Relax CSP for dynamic imports of view bundles (same agent origin only)
49
+ "Content-Security-Policy":
50
+ `default-src 'self' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +
51
+ `script-src 'self' 'unsafe-inline' 'unsafe-eval' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +
52
+ `style-src 'self' 'unsafe-inline'; ` +
53
+ `connect-src 'self' ${agentOrigin} ws://localhost:*;`,
54
+ },
55
+ body: html,
56
+ };
57
+ },
58
+ };
59
+
60
+ // ─────────────────────────────────────────────────────────────────────────────
61
+
62
+ function buildHostPage(
63
+ viewId: string,
64
+ bundleUrl: string,
65
+ viewsApiUrl: string,
66
+ agentOrigin: string,
67
+ ): string {
68
+ return `<!DOCTYPE html>
69
+ <html lang="en" data-view-id="${viewId}">
70
+ <head>
71
+ <meta charset="utf-8" />
72
+ <meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
73
+ <title>XR – ${viewId}</title>
74
+ <style>
75
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0 }
76
+
77
+ :root {
78
+ --bg: #0d0d0f;
79
+ --surface: #18181b;
80
+ --border: rgba(255,255,255,0.08);
81
+ --text: #f4f4f5;
82
+ --muted: #a1a1aa;
83
+ --accent: #6366f1;
84
+ --radius: 12px;
85
+ --font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
86
+ }
87
+
88
+ html, body {
89
+ width: 100%; height: 100%;
90
+ background: var(--bg);
91
+ color: var(--text);
92
+ font-family: var(--font);
93
+ font-size: 18px; /* large for XR readability */
94
+ line-height: 1.5;
95
+ overflow: hidden;
96
+ }
97
+
98
+ #xr-shell {
99
+ display: flex;
100
+ flex-direction: column;
101
+ height: 100%;
102
+ position: relative;
103
+ }
104
+
105
+ /* XR header bar */
106
+ #xr-bar {
107
+ display: flex;
108
+ align-items: center;
109
+ gap: 10px;
110
+ padding: 8px 16px;
111
+ background: var(--surface);
112
+ border-bottom: 1px solid var(--border);
113
+ flex-shrink: 0;
114
+ user-select: none;
115
+ }
116
+ #xr-bar-title {
117
+ flex: 1;
118
+ font-weight: 600;
119
+ font-size: 1rem;
120
+ color: var(--text);
121
+ }
122
+ .xr-btn {
123
+ background: var(--border);
124
+ border: none;
125
+ border-radius: 8px;
126
+ color: var(--text);
127
+ cursor: pointer;
128
+ font-size: 1rem;
129
+ padding: 6px 12px;
130
+ transition: background 0.15s;
131
+ }
132
+ .xr-btn:hover { background: rgba(255,255,255,0.15) }
133
+
134
+ /* Voice indicator */
135
+ #voice-indicator {
136
+ display: none;
137
+ align-items: center;
138
+ gap: 6px;
139
+ padding: 4px 10px;
140
+ border-radius: 20px;
141
+ background: var(--accent);
142
+ font-size: 0.8rem;
143
+ font-weight: 600;
144
+ }
145
+ #voice-indicator.active { display: flex }
146
+ .voice-dot {
147
+ width: 8px; height: 8px;
148
+ border-radius: 50%;
149
+ background: #fff;
150
+ animation: pulse 1s ease-in-out infinite;
151
+ }
152
+ @keyframes pulse { 0%,100% { opacity: 1 } 50% { opacity: 0.4 } }
153
+
154
+ /* Content area */
155
+ #view-mount {
156
+ flex: 1;
157
+ overflow: auto;
158
+ position: relative;
159
+ }
160
+
161
+ /* Loading / error states */
162
+ #view-loader {
163
+ display: flex;
164
+ flex-direction: column;
165
+ align-items: center;
166
+ justify-content: center;
167
+ height: 100%;
168
+ gap: 12px;
169
+ color: var(--muted);
170
+ }
171
+ .spinner {
172
+ width: 36px; height: 36px;
173
+ border: 3px solid var(--border);
174
+ border-top-color: var(--accent);
175
+ border-radius: 50%;
176
+ animation: spin 0.8s linear infinite;
177
+ }
178
+ @keyframes spin { to { transform: rotate(360deg) } }
179
+
180
+ /* XR-friendly form overrides for injected views */
181
+ #view-mount input, #view-mount textarea, #view-mount select {
182
+ font-size: 1rem !important;
183
+ min-height: 44px;
184
+ }
185
+ #view-mount button {
186
+ min-height: 44px;
187
+ min-width: 44px;
188
+ }
189
+
190
+ /* Transcript toast */
191
+ #transcript-toast {
192
+ position: fixed;
193
+ bottom: 12px; left: 50%;
194
+ transform: translateX(-50%);
195
+ background: rgba(99,102,241,0.9);
196
+ color: #fff;
197
+ border-radius: 20px;
198
+ padding: 6px 16px;
199
+ font-size: 0.85rem;
200
+ pointer-events: none;
201
+ opacity: 0;
202
+ transition: opacity 0.2s;
203
+ white-space: nowrap;
204
+ max-width: 90vw;
205
+ overflow: hidden;
206
+ text-overflow: ellipsis;
207
+ }
208
+ #transcript-toast.show { opacity: 1 }
209
+ </style>
210
+ </head>
211
+ <body>
212
+ <div id="xr-shell">
213
+ <div id="xr-bar">
214
+ <span id="xr-bar-title">${viewId}</span>
215
+ <div id="voice-indicator">
216
+ <div class="voice-dot"></div>
217
+ <span>Listening</span>
218
+ </div>
219
+ <button class="xr-btn" id="btn-close" title="Close panel">✕</button>
220
+ </div>
221
+
222
+ <div id="view-mount">
223
+ <div id="view-loader">
224
+ <div class="spinner"></div>
225
+ <span>Loading ${viewId}…</span>
226
+ </div>
227
+ </div>
228
+ </div>
229
+
230
+ <div id="transcript-toast"></div>
231
+
232
+ <!-- React from CDN — must match the version view bundles are built against -->
233
+ <script type="importmap">
234
+ {
235
+ "imports": {
236
+ "react": "https://esm.sh/react@18",
237
+ "react-dom": "https://esm.sh/react-dom@18",
238
+ "react-dom/client": "https://esm.sh/react-dom@18/client",
239
+ "react/jsx-runtime": "https://esm.sh/react@18/jsx-runtime"
240
+ }
241
+ }
242
+ </script>
243
+
244
+ <script type="module">
245
+ const VIEW_ID = "${viewId}";
246
+ const BUNDLE_URL = "${bundleUrl}";
247
+ const VIEWS_API = "${viewsApiUrl}";
248
+ const AGENT_ORIGIN = "${agentOrigin}";
249
+
250
+ // ── postMessage bridge ───────────────────────────────────────────────────
251
+
252
+ function notifyParent(msg) {
253
+ window.parent.postMessage(msg, "*");
254
+ }
255
+
256
+ window.addEventListener("message", (ev) => {
257
+ if (ev.data?.type === "xr:transcript") fillFocusedInput(ev.data.text);
258
+ if (ev.data?.type === "xr:focus-next") focusNext();
259
+ if (ev.data?.type === "xr:voice-start") showVoiceIndicator(true);
260
+ if (ev.data?.type === "xr:voice-end") showVoiceIndicator(false);
261
+ });
262
+
263
+ // ── Voice input helpers ──────────────────────────────────────────────────
264
+
265
+ function fillFocusedInput(text) {
266
+ const el = document.activeElement;
267
+ if (!el) return;
268
+ const tag = el.tagName;
269
+ if (tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT") {
270
+ const native = Object.getOwnPropertyDescriptor(window[tag === "INPUT" ? "HTMLInputElement" : tag === "TEXTAREA" ? "HTMLTextAreaElement" : "HTMLSelectElement"].prototype, "value");
271
+ native.set.call(el, text);
272
+ el.dispatchEvent(new Event("input", { bubbles: true }));
273
+ el.dispatchEvent(new Event("change", { bubbles: true }));
274
+ showTranscript(text);
275
+ } else if (el.getAttribute("role") === "combobox" || el.getAttribute("role") === "listbox") {
276
+ el.dispatchEvent(new CustomEvent("xr:voice-select", { bubbles: true, detail: { text } }));
277
+ showTranscript(text);
278
+ }
279
+ }
280
+
281
+ function focusNext() {
282
+ const all = Array.from(document.querySelectorAll("input,textarea,button,select,[tabindex]"))
283
+ .filter(el => !el.disabled && !el.closest("[hidden]"));
284
+ const idx = all.indexOf(document.activeElement);
285
+ const next = all[idx + 1] ?? all[0];
286
+ next?.focus();
287
+ }
288
+
289
+ function showVoiceIndicator(active) {
290
+ const el = document.getElementById("voice-indicator");
291
+ if (el) el.classList.toggle("active", active);
292
+ }
293
+
294
+ function showTranscript(text) {
295
+ const toast = document.getElementById("transcript-toast");
296
+ if (!toast) return;
297
+ toast.textContent = text;
298
+ toast.classList.add("show");
299
+ clearTimeout(toast._t);
300
+ toast._t = setTimeout(() => toast.classList.remove("show"), 3000);
301
+ }
302
+
303
+ // ── Close button ─────────────────────────────────────────────────────────
304
+
305
+ document.getElementById("btn-close")?.addEventListener("click", () => {
306
+ notifyParent({ type: "xr:close", viewId: VIEW_ID });
307
+ });
308
+
309
+ // ── Mount the view ───────────────────────────────────────────────────────
310
+
311
+ async function mountView() {
312
+ const mount = document.getElementById("view-mount");
313
+ const loader = document.getElementById("view-loader");
314
+
315
+ try {
316
+ // Provide minimal elizaOS-like context so views can render
317
+ window.__elizaXRContext = {
318
+ agentBaseUrl: AGENT_ORIGIN,
319
+ viewId: VIEW_ID,
320
+ fetchViews: () => fetch(VIEWS_API).then(r => r.json()),
321
+ navigate: (id) => notifyParent({ type: "xr:navigate", viewId: id }),
322
+ };
323
+
324
+ const mod = await import(/* @vite-ignore */ BUNDLE_URL);
325
+ const component = mod.default ?? mod[Object.keys(mod)[0]];
326
+
327
+ if (!component) throw new Error("No component export found in bundle");
328
+
329
+ // Dynamically import React + ReactDOM
330
+ const [React, ReactDOMClient] = await Promise.all([
331
+ import("react"),
332
+ import("react-dom/client"),
333
+ ]);
334
+
335
+ if (loader) loader.style.display = "none";
336
+
337
+ // Render the view component
338
+ const root = ReactDOMClient.createRoot(mount);
339
+ root.render(React.createElement(component));
340
+
341
+ notifyParent({ type: "xr:view-ready", viewId: VIEW_ID });
342
+
343
+ } catch (err) {
344
+ console.error("[xr-host] Failed to mount view:", err);
345
+ if (loader) loader.innerHTML =
346
+ \`<div style="color:#f87171;text-align:center;padding:24px">
347
+ <div style="font-size:1.5rem;margin-bottom:8px">⚠ Load error</div>
348
+ <div style="font-size:0.85rem;color:#a1a1aa">\${err.message}</div>
349
+ <button class="xr-btn" style="margin-top:16px" onclick="mountView()">Retry</button>
350
+ </div>\`;
351
+ notifyParent({ type: "xr:view-error", viewId: VIEW_ID, error: err.message });
352
+ }
353
+ }
354
+
355
+ mountView();
356
+ </script>
357
+ </body>
358
+ </html>`;
359
+ }
package/src/routes/xr-views.ts ADDED
@@ -0,0 +1,43 @@
1
+ import { listViews } from "@elizaos/agent/api/views-registry";
2
+ import type { Route } from "@elizaos/core";
3
+ import {
4
+ XR_SERVICE_TYPE,
5
+ type XRSessionService,
6
+ } from "../services/xr-session-service.ts";
7
+
8
+ /**
9
+ * GET /api/xr/views
10
+ * Returns all XR-typed views from registered plugins.
11
+ * Used by app-xr to populate the view launcher.
12
+ */
13
+ export const xrViewsRoute: Route = {
14
+ type: "GET",
15
+ path: "/xr/views",
16
+ description: "Lists all XR-capable views from registered plugins",
17
+ routeHandler: async (ctx) => {
18
+ const views = listViews({ developerMode: true, viewType: "xr" })
19
+ .filter((v) => v.viewType === "xr")
20
+ .map((v) => ({
21
+ id: v.id,
22
+ label: v.label,
23
+ icon: v.icon,
24
+ description: v.description,
25
+ tags: v.tags,
26
+ xrOptions: v.xrOptions,
27
+ path: v.path,
28
+ pluginName: v.pluginName,
29
+ available: v.available,
30
+ }));
31
+
32
+ const connections =
33
+ ctx.runtime
34
+ .getService<XRSessionService>(XR_SERVICE_TYPE)
35
+ ?.getConnections()
36
+ .map((c) => ({ id: c.id, deviceType: c.deviceType })) ?? [];
37
+
38
+ return {
39
+ status: 200,
40
+ body: { views, connections, count: views.length },
41
+ };
42
+ },
43
+ };
package/src/services/audio-pipeline.ts ADDED
@@ -0,0 +1,120 @@
1
+ import type { IAgentRuntime } from "@elizaos/core";
2
+ import { ModelType } from "@elizaos/core";
3
+ import type { XRAudioHeader } from "../protocol.ts";
4
+
5
+ // Prepend a RIFF/WAV header so Whisper can decode raw Float32 PCM.
6
+ function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
7
+ const channels = 1; // ScriptProcessorNode fallback is always mono
8
+ const dataSize = pcmData.length;
9
+ const header = Buffer.alloc(44);
10
+ header.write("RIFF", 0);
11
+ header.writeUInt32LE(36 + dataSize, 4);
12
+ header.write("WAVE", 8);
13
+ header.write("fmt ", 12);
14
+ header.writeUInt32LE(16, 16);
15
+ header.writeUInt16LE(3, 20); // IEEE_FLOAT
16
+ header.writeUInt16LE(channels, 22);
17
+ header.writeUInt32LE(sampleRate, 24);
18
+ header.writeUInt32LE(sampleRate * channels * 4, 28);
19
+ header.writeUInt16LE(channels * 4, 32);
20
+ header.writeUInt16LE(32, 34);
21
+ header.write("data", 36);
22
+ header.writeUInt32LE(dataSize, 40);
23
+ return Buffer.concat([header, pcmData]);
24
+ }
25
+
26
+ // Accumulate up to FLUSH_AFTER_MS of audio then transcribe.
27
+ // Also flush if no chunk arrives within SILENCE_GAP_MS (end-of-utterance).
28
+ const FLUSH_AFTER_MS = 2000;
29
+ const SILENCE_GAP_MS = 1500;
30
+
31
+ export interface PendingTranscription {
32
+ chunks: Buffer[];
33
+ firstTs: number;
34
+ lastTs: number;
35
+ encoding: XRAudioHeader["encoding"];
36
+ sampleRate: number;
37
+ silenceTimer?: ReturnType<typeof setTimeout>;
38
+ }
39
+
40
+ export class AudioPipeline {
41
+ private pending = new Map<string, PendingTranscription>();
42
+
43
+ constructor(
44
+ private readonly runtime: IAgentRuntime,
45
+ private readonly onTranscript: (
46
+ connectionId: string,
47
+ text: string,
48
+ ) => Promise<void>,
49
+ ) {}
50
+
51
+ push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void {
52
+ let state = this.pending.get(connectionId);
53
+ if (!state) {
54
+ state = {
55
+ chunks: [],
56
+ firstTs: header.ts,
57
+ lastTs: header.ts,
58
+ encoding: header.encoding,
59
+ sampleRate: header.sampleRate,
60
+ };
61
+ this.pending.set(connectionId, state);
62
+ }
63
+
64
+ state.chunks.push(chunk);
65
+ state.lastTs = header.ts;
66
+
67
+ // Reset silence timer on each incoming chunk
68
+ if (state.silenceTimer) clearTimeout(state.silenceTimer);
69
+ state.silenceTimer = setTimeout(
70
+ () => void this.flush(connectionId),
71
+ SILENCE_GAP_MS,
72
+ );
73
+
74
+ // Also flush if we've accumulated enough audio
75
+ if (state.lastTs - state.firstTs >= FLUSH_AFTER_MS) {
76
+ void this.flush(connectionId);
77
+ }
78
+ }
79
+
80
+ async flush(connectionId: string): Promise<void> {
81
+ const state = this.pending.get(connectionId);
82
+ if (!state || state.chunks.length === 0) return;
83
+
84
+ if (state.silenceTimer) {
85
+ clearTimeout(state.silenceTimer);
86
+ state.silenceTimer = undefined;
87
+ }
88
+ this.pending.delete(connectionId);
89
+
90
+ const combined = Buffer.concat(state.chunks);
91
+ if (combined.length < 512) return; // too small to be real speech
92
+
93
+ // pcm-f32 is raw Float32 samples (mono, from ScriptProcessorNode fallback).
94
+ // Whisper expects a valid audio container — wrap with a WAV header.
95
+ const audioBuffer =
96
+ state.encoding === "pcm-f32"
97
+ ? pcmF32ToWav(combined, state.sampleRate)
98
+ : combined;
99
+
100
+ try {
101
+ const transcript = await this.runtime.useModel(
102
+ ModelType.TRANSCRIPTION,
103
+ audioBuffer,
104
+ );
105
+ const text = typeof transcript === "string" ? transcript.trim() : "";
106
+ if (text.length > 0) {
107
+ await this.onTranscript(connectionId, text);
108
+ }
109
+ } catch (err) {
110
+ // log but don't crash the pipeline
111
+ console.error("[plugin-xr] transcription error:", err);
112
+ }
113
+ }
114
+
115
+ clear(connectionId: string): void {
116
+ const state = this.pending.get(connectionId);
117
+ if (state?.silenceTimer) clearTimeout(state.silenceTimer);
118
+ this.pending.delete(connectionId);
119
+ }
120
+ }