@elizaos/plugin-xr 2.0.3-beta.6 → 2.0.3-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/xr-query-vision.d.ts +3 -0
- package/dist/actions/xr-query-vision.d.ts.map +1 -0
- package/dist/actions/xr-query-vision.js +39 -0
- package/dist/actions/xr-query-vision.js.map +1 -0
- package/dist/actions/xr-view-actions.d.ts +18 -0
- package/dist/actions/xr-view-actions.d.ts.map +1 -0
- package/dist/actions/xr-view-actions.js +304 -0
- package/dist/actions/xr-view-actions.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +57 -0
- package/dist/index.js.map +1 -0
- package/dist/protocol.d.ts +124 -0
- package/dist/protocol.d.ts.map +1 -0
- package/dist/protocol.js +18 -0
- package/dist/protocol.js.map +1 -0
- package/dist/providers/xr-context.d.ts +3 -0
- package/dist/providers/xr-context.d.ts.map +1 -0
- package/dist/providers/xr-context.js +34 -0
- package/dist/providers/xr-context.js.map +1 -0
- package/dist/routes/xr-connect.d.ts +3 -0
- package/dist/routes/xr-connect.d.ts.map +1 -0
- package/{src/routes/xr-connect.ts → dist/routes/xr-connect.js} +12 -15
- package/dist/routes/xr-connect.js.map +1 -0
- package/dist/routes/xr-simulator-route.d.ts +8 -0
- package/dist/routes/xr-simulator-route.d.ts.map +1 -0
- package/{src/routes/xr-simulator-route.ts → dist/routes/xr-simulator-route.js} +10 -16
- package/dist/routes/xr-simulator-route.js.map +1 -0
- package/dist/routes/xr-status.d.ts +3 -0
- package/dist/routes/xr-status.d.ts.map +1 -0
- package/{src/routes/xr-status.ts → dist/routes/xr-status.js} +13 -15
- package/dist/routes/xr-status.js.map +1 -0
- package/dist/routes/xr-view-host.d.ts +24 -0
- package/dist/routes/xr-view-host.d.ts.map +1 -0
- package/{src/routes/xr-view-host.ts → dist/routes/xr-view-host.js} +22 -59
- package/dist/routes/xr-view-host.js.map +1 -0
- package/dist/routes/xr-views.d.ts +8 -0
- package/dist/routes/xr-views.d.ts.map +1 -0
- package/dist/routes/xr-views.js +31 -0
- package/dist/routes/xr-views.js.map +1 -0
- package/dist/services/audio-pipeline.d.ts +20 -0
- package/dist/services/audio-pipeline.d.ts.map +1 -0
- package/{src/services/audio-pipeline.ts → dist/services/audio-pipeline.js} +25 -58
- package/dist/services/audio-pipeline.js.map +1 -0
- package/dist/services/vision-pipeline.d.ts +16 -0
- package/dist/services/vision-pipeline.d.ts.map +1 -0
- package/dist/services/vision-pipeline.js +39 -0
- package/dist/services/vision-pipeline.js.map +1 -0
- package/dist/services/xr-session-service.d.ts +50 -0
- package/dist/services/xr-session-service.d.ts.map +1 -0
- package/{src/services/xr-session-service.ts → dist/services/xr-session-service.js} +85 -194
- package/dist/services/xr-session-service.js.map +1 -0
- package/package.json +9 -4
- package/AGENTS.md +0 -151
- package/CLAUDE.md +0 -151
- package/simulator/bun.lock +0 -159
- package/simulator/package.json +0 -28
- package/simulator/src/emulator.ts +0 -174
- package/simulator/src/mock-agent.ts +0 -233
- package/simulator/src/node.ts +0 -9
- package/simulator/src/playwright-fixture.ts +0 -169
- package/simulator/src/types.ts +0 -51
- package/simulator/tsconfig.json +0 -13
- package/simulator/vite.config.ts +0 -25
- package/src/__tests__/audio-pipeline.test.ts +0 -129
- package/src/__tests__/protocol.test.ts +0 -53
- package/src/__tests__/routes-e2e.test.ts +0 -276
- package/src/__tests__/vision-pipeline.test.ts +0 -73
- package/src/__tests__/xr-bundle-coverage.test.ts +0 -303
- package/src/__tests__/xr-feature-parity.test.ts +0 -524
- package/src/__tests__/xr-functional-parity.test.ts +0 -522
- package/src/__tests__/xr-view-host-http.test.ts +0 -239
- package/src/__tests__/xr-view-host.test.ts +0 -174
- package/src/actions/xr-query-vision.ts +0 -64
- package/src/actions/xr-view-actions.ts +0 -386
- package/src/index.ts +0 -55
- package/src/protocol.ts +0 -126
- package/src/providers/xr-context.ts +0 -49
- package/src/routes/xr-views.ts +0 -43
- package/src/services/vision-pipeline.ts +0 -57
- package/tsconfig.build.json +0 -9
- package/tsconfig.json +0 -30
- package/vitest.config.ts +0 -21
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/protocol.ts"],"sourcesContent":["// Binary frame layout:\n// bytes 0–3 : big-endian uint32 — JSON header length\n// bytes 4–N : UTF-8 JSON header\n// bytes N+1… : raw binary payload (audio PCM/Opus, JPEG, etc.)\n//\n// Text frames are JSON control messages (no binary payload).\n\nexport type XRDeviceType = \"quest3\" | \"xreal\" | \"simulator\";\n\n// ── Client → Server (text frames) ──────────────────────────────────────────\n\n/** View panel state reported back from the XR device */\nexport interface XRViewPanelState {\n viewId: string;\n active: boolean;\n width?: number;\n height?: number;\n}\n\nexport type XRClientControl =\n | { type: \"hello\"; deviceType: XRDeviceType; sessionId: string }\n | { type: \"ping\" }\n | { type: \"view_ready\"; viewId: string }\n | { type: \"view_closed\"; viewId: string }\n | { type: \"view_event\"; viewId: string; event: string; payload?: unknown };\n\n// ── Client → Server (binary frames) ────────────────────────────────────────\n\nexport interface XRAudioHeader {\n type: \"audio\";\n ts: number;\n sampleRate: number;\n /** \"webm-opus\" from MediaRecorder, \"pcm-f32\" from ScriptProcessor fallback */\n encoding: \"webm-opus\" | \"pcm-f32\";\n}\n\nexport interface XRFrameHeader {\n type: \"frame\";\n ts: number;\n width: number;\n height: number;\n format: \"jpeg\" | \"webp\";\n pose?: {\n position: { x: number; y: number; z: number };\n orientation: { x: number; y: number; z: number; w: number };\n };\n}\n\nexport type XRBinaryHeader = XRAudioHeader | XRFrameHeader;\n\n// ── Server → Client (text frames) ──────────────────────────────────────────\n\n/** XR panel sizing options */\nexport interface XRPanelConfig {\n /** Panel width relative to default (0.5 = half, 2.0 = double) */\n scale?: number;\n /** Follow mode: billboard | fixed | follow */\n followMode?: \"billboard\" | \"fixed\" | \"follow\";\n /** Distance from camera in metres */\n distance?: number;\n /** Whether to show as full-overlay or floating panel */\n fullscreen?: boolean;\n}\n\nexport type XRServerControl =\n | { type: \"ready\"; sessionId: string }\n | { type: \"transcript\"; text: string; final: boolean }\n | { type: \"agent_text\"; text: string }\n | { type: \"pong\" }\n // ── View commands ────────────────────────────────────────────────────────\n /** Open (or bring to front) a view by its registered view id */\n | {\n type: \"view_open\";\n viewId: string;\n agentBaseUrl: string;\n config?: XRPanelConfig;\n }\n /** Close a specific view panel */\n | { type: \"view_close\"; viewId: string }\n /** Switch the \"active\" (foreground) view */\n | { type: \"view_switch\"; viewId: string }\n /** Resize / reposition the active or named panel */\n | { type: \"view_resize\"; viewId?: string; config: XRPanelConfig }\n /** Send all available views to the device for the launcher */\n | {\n type: \"views_catalog\";\n views: Array<{\n id: string;\n label: string;\n icon?: string;\n description?: string;\n }>;\n };\n\n// ── Server → Client (binary frames) ────────────────────────────────────────\n\nexport interface XRTTSAudioHeader {\n type: \"tts_audio\";\n sampleRate: number;\n channels: number;\n /** encoding of the outbound audio */\n encoding: \"mp3\" | \"wav\" | \"pcm-f32\";\n}\n\n// ── Framing helpers ─────────────────────────────────────────────────────────\n\nexport function encodeBinaryFrame(\n header: XRBinaryHeader | XRTTSAudioHeader,\n payload: Uint8Array | Buffer,\n): Buffer {\n const headerJson = Buffer.from(JSON.stringify(header), \"utf8\");\n const lenBuf = Buffer.allocUnsafe(4);\n lenBuf.writeUInt32BE(headerJson.length, 0);\n return Buffer.concat([lenBuf, headerJson, payload]);\n}\n\nexport function decodeBinaryFrame(data: Buffer): {\n header: XRBinaryHeader | XRTTSAudioHeader;\n payload: Buffer;\n} {\n const headerLen = data.readUInt32BE(0);\n const headerJson = data.subarray(4, 4 + headerLen).toString(\"utf8\");\n const header = JSON.parse(headerJson) as XRBinaryHeader | XRTTSAudioHeader;\n const payload = data.subarray(4 + headerLen);\n return { header, payload };\n}\n"],"mappings":"AA0GO,SAAS,kBACd,QACA,SACQ;AACR,QAAM,aAAa,OAAO,KAAK,KAAK,UAAU,MAAM,GAAG,MAAM;AAC7D,QAAM,SAAS,OAAO,YAAY,CAAC;AACnC,SAAO,cAAc,WAAW,QAAQ,CAAC;AACzC,SAAO,OAAO,OAAO,CAAC,QAAQ,YAAY,OAAO,CAAC;AACpD;AAEO,SAAS,kBAAkB,MAGhC;AACA,QAAM,YAAY,KAAK,aAAa,CAAC;AACrC,QAAM,aAAa,KAAK,SAAS,GAAG,IAAI,SAAS,EAAE,SAAS,MAAM;AAClE,QAAM,SAAS,KAAK,MAAM,UAAU;AACpC,QAAM,UAAU,KAAK,SAAS,IAAI,SAAS;AAC3C,SAAO,EAAE,QAAQ,QAAQ;AAC3B;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-context.d.ts","sourceRoot":"","sources":["../../src/providers/xr-context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,QAAQ,EAGT,MAAM,eAAe,CAAC;AAMvB,eAAO,MAAM,iBAAiB,EAAE,QAoC/B,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import {
|
|
2
|
+
XR_SERVICE_TYPE
|
|
3
|
+
} from "../services/xr-session-service.js";
|
|
4
|
+
const xrContextProvider = {
|
|
5
|
+
name: "XR_SESSION",
|
|
6
|
+
description: "Provides context about connected XR headsets (Quest 3, XReal)",
|
|
7
|
+
get: async (runtime, _message, _state) => {
|
|
8
|
+
const svc = runtime.getService(XR_SERVICE_TYPE);
|
|
9
|
+
if (!svc?.hasActiveConnections()) return { text: "" };
|
|
10
|
+
const conns = svc.getConnections();
|
|
11
|
+
const deviceList = conns.map((c) => {
|
|
12
|
+
const hasFrame = svc.getVisionPipeline().hasRecentFrame(c.id);
|
|
13
|
+
return `${c.deviceType}${hasFrame ? " (camera active)" : ""}`;
|
|
14
|
+
}).join(", ");
|
|
15
|
+
const text = [
|
|
16
|
+
`[XR devices connected: ${deviceList}]`,
|
|
17
|
+
`[Audio streaming active \u2014 the user is speaking to you via their headset microphone.]`,
|
|
18
|
+
`[Your text responses will be spoken aloud through the headset via TTS.]`,
|
|
19
|
+
`[Use XR_QUERY_VISION to describe what the user's camera sees.]`
|
|
20
|
+
].join("\n");
|
|
21
|
+
return {
|
|
22
|
+
text,
|
|
23
|
+
values: {
|
|
24
|
+
xrConnected: true,
|
|
25
|
+
xrDevices: conns.map((c) => c.deviceType),
|
|
26
|
+
xrConnectionCount: conns.length
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
export {
|
|
32
|
+
xrContextProvider
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=xr-context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/providers/xr-context.ts"],"sourcesContent":["import type {\n IAgentRuntime,\n Memory,\n Provider,\n ProviderResult,\n State,\n} from \"@elizaos/core\";\nimport {\n XR_SERVICE_TYPE,\n type XRSessionService,\n} from \"../services/xr-session-service.js\";\n\nexport const xrContextProvider: Provider = {\n name: \"XR_SESSION\",\n description: \"Provides context about connected XR headsets (Quest 3, XReal)\",\n\n get: async (\n runtime: IAgentRuntime,\n _message: Memory,\n _state: State,\n ): Promise<ProviderResult> => {\n const svc = runtime.getService<XRSessionService>(XR_SERVICE_TYPE);\n if (!svc?.hasActiveConnections()) return { text: \"\" };\n\n const conns = svc.getConnections();\n const deviceList = conns\n .map((c) => {\n const hasFrame = svc.getVisionPipeline().hasRecentFrame(c.id);\n return `${c.deviceType}${hasFrame ? \" (camera active)\" : \"\"}`;\n })\n .join(\", \");\n\n const text = [\n `[XR devices connected: ${deviceList}]`,\n `[Audio streaming active — the user is speaking to you via their headset microphone.]`,\n `[Your text responses will be spoken aloud through the headset via TTS.]`,\n `[Use XR_QUERY_VISION to describe what the user's camera sees.]`,\n ].join(\"\\n\");\n\n return {\n text,\n values: {\n xrConnected: true,\n xrDevices: conns.map((c) => c.deviceType),\n xrConnectionCount: conns.length,\n },\n };\n },\n};\n"],"mappings":"AAOA;AAAA,EACE;AAAA,OAEK;AAEA,MAAM,oBAA8B;AAAA,EACzC,MAAM;AAAA,EACN,aAAa;AAAA,EAEb,KAAK,OACH,SACA,UACA,WAC4B;AAC5B,UAAM,MAAM,QAAQ,WAA6B,eAAe;AAChE,QAAI,CAAC,KAAK,qBAAqB,EAAG,QAAO,EAAE,MAAM,GAAG;AAEpD,UAAM,QAAQ,IAAI,eAAe;AACjC,UAAM,aAAa,MAChB,IAAI,CAAC,MAAM;AACV,YAAM,WAAW,IAAI,kBAAkB,EAAE,eAAe,EAAE,EAAE;AAC5D,aAAO,GAAG,EAAE,UAAU,GAAG,WAAW,qBAAqB,EAAE;AAAA,IAC7D,CAAC,EACA,KAAK,IAAI;AAEZ,UAAM,OAAO;AAAA,MACX,0BAA0B,UAAU;AAAA,MACpC;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAEX,WAAO;AAAA,MACL;AAAA,MACA,QAAQ;AAAA,QACN,aAAa;AAAA,QACb,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,UAAU;AAAA,QACxC,mBAAmB,MAAM;AAAA,MAC3B;AAAA,IACF;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-connect.d.ts","sourceRoot":"","sources":["../../src/routes/xr-connect.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AA0E3C,eAAO,MAAM,cAAc,EAAE,KAa5B,CAAC"}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { networkInterfaces } from "node:os";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
function getLocalIp(): string {
|
|
2
|
+
function getLocalIp() {
|
|
5
3
|
const nets = networkInterfaces();
|
|
6
4
|
for (const iface of Object.values(nets)) {
|
|
7
5
|
for (const net of iface ?? []) {
|
|
@@ -10,15 +8,12 @@ function getLocalIp(): string {
|
|
|
10
8
|
}
|
|
11
9
|
return "127.0.0.1";
|
|
12
10
|
}
|
|
13
|
-
|
|
14
|
-
function getAppUrl(): string {
|
|
15
|
-
// Set by the connect script when a tunnel is active
|
|
11
|
+
function getAppUrl() {
|
|
16
12
|
if (process.env.XR_APP_URL) return process.env.XR_APP_URL;
|
|
17
13
|
const port = process.env.VITE_PORT ?? "5173";
|
|
18
14
|
return `http://${getLocalIp()}:${port}`;
|
|
19
15
|
}
|
|
20
|
-
|
|
21
|
-
function htmlPage(appUrl: string): string {
|
|
16
|
+
function htmlPage(appUrl) {
|
|
22
17
|
const encoded = encodeURIComponent(appUrl);
|
|
23
18
|
return `<!DOCTYPE html>
|
|
24
19
|
<html lang="en">
|
|
@@ -56,7 +51,7 @@ function htmlPage(appUrl: string): string {
|
|
|
56
51
|
<li>Allow microphone and camera access when prompted</li>
|
|
57
52
|
<li>The agent will connect automatically</li>
|
|
58
53
|
</ol>
|
|
59
|
-
${appUrl.startsWith("http://") ? `<p class="warn"
|
|
54
|
+
${appUrl.startsWith("http://") ? `<p class="warn">\u26A0 HTTP URL \u2014 WebXR requires HTTPS on device.<br>Run <code>bun run connect</code> for an HTTPS tunnel.</p>` : ""}
|
|
60
55
|
</div>
|
|
61
56
|
<script src="https://cdn.jsdelivr.net/npm/qrcode@1.5.4/build/qrcode.min.js"></script>
|
|
62
57
|
<script>
|
|
@@ -72,18 +67,20 @@ function htmlPage(appUrl: string): string {
|
|
|
72
67
|
</body>
|
|
73
68
|
</html>`;
|
|
74
69
|
}
|
|
75
|
-
|
|
76
|
-
export const xrConnectRoute: Route = {
|
|
70
|
+
const xrConnectRoute = {
|
|
77
71
|
type: "GET",
|
|
78
72
|
path: "/xr/connect",
|
|
79
|
-
description:
|
|
80
|
-
"Returns an HTML page with a QR code to connect an XR headset. Set XR_APP_URL env var (or run `bun run connect` in plugins/plugin-facewear/app-xr) to show the correct public URL.",
|
|
73
|
+
description: "Returns an HTML page with a QR code to connect an XR headset. Set XR_APP_URL env var (or run `bun run connect` in plugins/plugin-facewear/app-xr) to show the correct public URL.",
|
|
81
74
|
routeHandler: async (_ctx) => {
|
|
82
75
|
const url = getAppUrl();
|
|
83
76
|
return {
|
|
84
77
|
status: 200,
|
|
85
78
|
headers: { "Content-Type": "text/html; charset=utf-8" },
|
|
86
|
-
body: htmlPage(url)
|
|
79
|
+
body: htmlPage(url)
|
|
87
80
|
};
|
|
88
|
-
}
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
export {
|
|
84
|
+
xrConnectRoute
|
|
89
85
|
};
|
|
86
|
+
//# sourceMappingURL=xr-connect.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/routes/xr-connect.ts"],"sourcesContent":["import { networkInterfaces } from \"node:os\";\nimport type { Route } from \"@elizaos/core\";\n\nfunction getLocalIp(): string {\n const nets = networkInterfaces();\n for (const iface of Object.values(nets)) {\n for (const net of iface ?? []) {\n if (!net.internal && net.family === \"IPv4\") return net.address;\n }\n }\n return \"127.0.0.1\";\n}\n\nfunction getAppUrl(): string {\n // Set by the connect script when a tunnel is active\n if (process.env.XR_APP_URL) return process.env.XR_APP_URL;\n const port = process.env.VITE_PORT ?? \"5173\";\n return `http://${getLocalIp()}:${port}`;\n}\n\nfunction htmlPage(appUrl: string): string {\n const encoded = encodeURIComponent(appUrl);\n return `<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"utf-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n <title>Connect XR Headset</title>\n <style>\n * { box-sizing: border-box; margin: 0; padding: 0 }\n body { font-family: system-ui, sans-serif; background: #0f0f0f; color: #e5e5e5;\n min-height: 100vh; display: flex; align-items: center; justify-content: center; }\n .card { background: #1a1a1a; border: 1px solid #2a2a2a; border-radius: 16px;\n padding: 40px; max-width: 480px; width: 100%; text-align: center; }\n h1 { font-size: 1.5rem; margin-bottom: 8px; }\n .sub { color: #888; font-size: 0.9rem; margin-bottom: 28px; }\n #qrcanvas { border-radius: 12px; background: white; padding: 12px;\n display: block; margin: 0 auto 24px; }\n .url { background: #111; border: 1px solid #333; border-radius: 8px;\n padding: 10px 16px; font-family: monospace; font-size: 0.85rem;\n word-break: break-all; margin-bottom: 24px; }\n .steps { text-align: left; font-size: 0.85rem; color: #aaa; line-height: 1.7; }\n .steps li { margin-bottom: 4px; }\n .warn { margin-top: 20px; font-size: 0.78rem; color: #f59e0b; }\n </style>\n</head>\n<body>\n <div class=\"card\">\n <h1>Connect XR Headset</h1>\n <p class=\"sub\">Scan to open the XR app on your device</p>\n <canvas id=\"qrcanvas\"></canvas>\n <div class=\"url\">${appUrl}</div>\n <ol class=\"steps\">\n <li>Put on your headset and open the browser</li>\n <li>Scan the QR code or type the URL above</li>\n <li>Allow microphone and camera access when prompted</li>\n <li>The agent will connect automatically</li>\n </ol>\n ${appUrl.startsWith(\"http://\") ? `<p class=\"warn\">⚠ HTTP URL — WebXR requires HTTPS on device.<br>Run <code>bun run connect</code> for an HTTPS tunnel.</p>` : \"\"}\n </div>\n <script src=\"https://cdn.jsdelivr.net/npm/qrcode@1.5.4/build/qrcode.min.js\"></script>\n <script>\n var url = decodeURIComponent(\"${encoded}\");\n if (typeof QRCode !== \"undefined\") {\n QRCode.toCanvas(document.getElementById(\"qrcanvas\"), url, { width: 220, margin: 2 }, function(err) {\n if (err) document.getElementById(\"qrcanvas\").style.display = \"none\";\n });\n } else {\n document.getElementById(\"qrcanvas\").style.display = \"none\";\n }\n </script>\n</body>\n</html>`;\n}\n\nexport const xrConnectRoute: Route = {\n type: \"GET\",\n path: \"/xr/connect\",\n description:\n \"Returns an HTML page with a QR code to connect an XR headset. Set XR_APP_URL env var (or run `bun run connect` in plugins/plugin-facewear/app-xr) to show the correct public URL.\",\n routeHandler: async (_ctx) => {\n const url = getAppUrl();\n return {\n status: 200,\n headers: { \"Content-Type\": \"text/html; charset=utf-8\" },\n body: htmlPage(url),\n };\n },\n};\n"],"mappings":"AAAA,SAAS,yBAAyB;AAGlC,SAAS,aAAqB;AAC5B,QAAM,OAAO,kBAAkB;AAC/B,aAAW,SAAS,OAAO,OAAO,IAAI,GAAG;AACvC,eAAW,OAAO,SAAS,CAAC,GAAG;AAC7B,UAAI,CAAC,IAAI,YAAY,IAAI,WAAW,OAAQ,QAAO,IAAI;AAAA,IACzD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,YAAoB;AAE3B,MAAI,QAAQ,IAAI,WAAY,QAAO,QAAQ,IAAI;AAC/C,QAAM,OAAO,QAAQ,IAAI,aAAa;AACtC,SAAO,UAAU,WAAW,CAAC,IAAI,IAAI;AACvC;AAEA,SAAS,SAAS,QAAwB;AACxC,QAAM,UAAU,mBAAmB,MAAM;AACzC,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBA6Bc,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOvB,OAAO,WAAW,SAAS,IAAI,wIAA8H,EAAE;AAAA;AAAA;AAAA;AAAA,oCAIjI,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAW3C;AAEO,MAAM,iBAAwB;AAAA,EACnC,MAAM;AAAA,EACN,MAAM;AAAA,EACN,aACE;AAAA,EACF,cAAc,OAAO,SAAS;AAC5B,UAAM,MAAM,UAAU;AACtB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,2BAA2B;AAAA,MACtD,MAAM,SAAS,GAAG;AAAA,IACpB;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Route } from "@elizaos/core";
|
|
2
|
+
/**
|
|
3
|
+
* Serves the built WebXR emulator bundle at GET /api/xr/simulator.js
|
|
4
|
+
* Only active when the bundle exists (i.e., after `bun run simulator:build`).
|
|
5
|
+
* Used by Playwright tests that prefer HTTP-served scripts over filesystem paths.
|
|
6
|
+
*/
|
|
7
|
+
export declare const xrSimulatorRoute: Route;
|
|
8
|
+
//# sourceMappingURL=xr-simulator-route.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-simulator-route.d.ts","sourceRoot":"","sources":["../../src/routes/xr-simulator-route.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAK3C;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,EAAE,KAuB9B,CAAC"}
|
|
@@ -1,37 +1,31 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { dirname, resolve } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import type { Route } from "@elizaos/core";
|
|
5
|
-
|
|
6
4
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
7
5
|
const EMULATOR_BUNDLE = resolve(__dirname, "../../simulator/dist/emulator.js");
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Serves the built WebXR emulator bundle at GET /api/xr/simulator.js
|
|
11
|
-
* Only active when the bundle exists (i.e., after `bun run simulator:build`).
|
|
12
|
-
* Used by Playwright tests that prefer HTTP-served scripts over filesystem paths.
|
|
13
|
-
*/
|
|
14
|
-
export const xrSimulatorRoute: Route = {
|
|
6
|
+
const xrSimulatorRoute = {
|
|
15
7
|
type: "GET",
|
|
16
8
|
path: "/xr/simulator.js",
|
|
17
|
-
description:
|
|
18
|
-
"Serves the XR device emulator bundle (IWER + camera injection) for Playwright testing. Build first: cd plugins/plugin-xr/simulator && bun run build",
|
|
9
|
+
description: "Serves the XR device emulator bundle (IWER + camera injection) for Playwright testing. Build first: cd plugins/plugin-xr/simulator && bun run build",
|
|
19
10
|
routeHandler: async (_ctx) => {
|
|
20
11
|
if (!existsSync(EMULATOR_BUNDLE)) {
|
|
21
12
|
return {
|
|
22
13
|
status: 404,
|
|
23
14
|
body: {
|
|
24
15
|
error: "Emulator bundle not built",
|
|
25
|
-
hint: "Run: cd eliza/plugins/plugin-xr/simulator && bun run build"
|
|
26
|
-
}
|
|
16
|
+
hint: "Run: cd eliza/plugins/plugin-xr/simulator && bun run build"
|
|
17
|
+
}
|
|
27
18
|
};
|
|
28
19
|
}
|
|
29
|
-
|
|
30
20
|
const js = readFileSync(EMULATOR_BUNDLE, "utf8");
|
|
31
21
|
return {
|
|
32
22
|
status: 200,
|
|
33
23
|
headers: { "Content-Type": "application/javascript; charset=utf-8" },
|
|
34
|
-
body: js
|
|
24
|
+
body: js
|
|
35
25
|
};
|
|
36
|
-
}
|
|
26
|
+
}
|
|
37
27
|
};
|
|
28
|
+
export {
|
|
29
|
+
xrSimulatorRoute
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=xr-simulator-route.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/routes/xr-simulator-route.ts"],"sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { dirname, resolve } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport type { Route } from \"@elizaos/core\";\n\nconst __dirname = dirname(fileURLToPath(import.meta.url));\nconst EMULATOR_BUNDLE = resolve(__dirname, \"../../simulator/dist/emulator.js\");\n\n/**\n * Serves the built WebXR emulator bundle at GET /api/xr/simulator.js\n * Only active when the bundle exists (i.e., after `bun run simulator:build`).\n * Used by Playwright tests that prefer HTTP-served scripts over filesystem paths.\n */\nexport const xrSimulatorRoute: Route = {\n type: \"GET\",\n path: \"/xr/simulator.js\",\n description:\n \"Serves the XR device emulator bundle (IWER + camera injection) for Playwright testing. Build first: cd plugins/plugin-xr/simulator && bun run build\",\n routeHandler: async (_ctx) => {\n if (!existsSync(EMULATOR_BUNDLE)) {\n return {\n status: 404,\n body: {\n error: \"Emulator bundle not built\",\n hint: \"Run: cd eliza/plugins/plugin-xr/simulator && bun run build\",\n },\n };\n }\n\n const js = readFileSync(EMULATOR_BUNDLE, \"utf8\");\n return {\n status: 200,\n headers: { \"Content-Type\": \"application/javascript; charset=utf-8\" },\n body: js,\n };\n },\n};\n"],"mappings":"AAAA,SAAS,YAAY,oBAAoB;AACzC,SAAS,SAAS,eAAe;AACjC,SAAS,qBAAqB;AAG9B,MAAM,YAAY,QAAQ,cAAc,YAAY,GAAG,CAAC;AACxD,MAAM,kBAAkB,QAAQ,WAAW,kCAAkC;AAOtE,MAAM,mBAA0B;AAAA,EACrC,MAAM;AAAA,EACN,MAAM;AAAA,EACN,aACE;AAAA,EACF,cAAc,OAAO,SAAS;AAC5B,QAAI,CAAC,WAAW,eAAe,GAAG;AAChC,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,MAAM;AAAA,UACJ,OAAO;AAAA,UACP,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,KAAK,aAAa,iBAAiB,MAAM;AAC/C,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,wCAAwC;AAAA,MACnE,MAAM;AAAA,IACR;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-status.d.ts","sourceRoot":"","sources":["../../src/routes/xr-status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAM3C,eAAO,MAAM,aAAa,EAAE,KA6B3B,CAAC"}
|
|
@@ -1,36 +1,34 @@
|
|
|
1
|
-
import type { Route } from "@elizaos/core";
|
|
2
1
|
import {
|
|
3
|
-
XR_SERVICE_TYPE
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export const xrStatusRoute: Route = {
|
|
2
|
+
XR_SERVICE_TYPE
|
|
3
|
+
} from "../services/xr-session-service.js";
|
|
4
|
+
const xrStatusRoute = {
|
|
8
5
|
type: "GET",
|
|
9
6
|
path: "/xr/status",
|
|
10
7
|
description: "Returns the list of connected XR devices and session state",
|
|
11
8
|
routeHandler: async (ctx) => {
|
|
12
|
-
const svc = ctx.runtime.getService
|
|
13
|
-
|
|
9
|
+
const svc = ctx.runtime.getService(XR_SERVICE_TYPE);
|
|
14
10
|
if (!svc) {
|
|
15
11
|
return {
|
|
16
12
|
status: 503,
|
|
17
|
-
body: { error: "XR service not running" }
|
|
13
|
+
body: { error: "XR service not running" }
|
|
18
14
|
};
|
|
19
15
|
}
|
|
20
|
-
|
|
21
16
|
const conns = svc.getConnections().map((c) => ({
|
|
22
17
|
id: c.id,
|
|
23
18
|
deviceType: c.deviceType,
|
|
24
19
|
connectedAt: c.connectedAt.toISOString(),
|
|
25
|
-
hasRecentFrame: svc.getVisionPipeline().hasRecentFrame(c.id)
|
|
20
|
+
hasRecentFrame: svc.getVisionPipeline().hasRecentFrame(c.id)
|
|
26
21
|
}));
|
|
27
|
-
|
|
28
22
|
return {
|
|
29
23
|
status: 200,
|
|
30
24
|
body: {
|
|
31
25
|
connected: conns.length > 0,
|
|
32
|
-
connections: conns
|
|
33
|
-
}
|
|
26
|
+
connections: conns
|
|
27
|
+
}
|
|
34
28
|
};
|
|
35
|
-
}
|
|
29
|
+
}
|
|
36
30
|
};
|
|
31
|
+
export {
|
|
32
|
+
xrStatusRoute
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=xr-status.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/routes/xr-status.ts"],"sourcesContent":["import type { Route } from \"@elizaos/core\";\nimport {\n XR_SERVICE_TYPE,\n type XRSessionService,\n} from \"../services/xr-session-service.js\";\n\nexport const xrStatusRoute: Route = {\n type: \"GET\",\n path: \"/xr/status\",\n description: \"Returns the list of connected XR devices and session state\",\n routeHandler: async (ctx) => {\n const svc = ctx.runtime.getService<XRSessionService>(XR_SERVICE_TYPE);\n\n if (!svc) {\n return {\n status: 503,\n body: { error: \"XR service not running\" },\n };\n }\n\n const conns = svc.getConnections().map((c) => ({\n id: c.id,\n deviceType: c.deviceType,\n connectedAt: c.connectedAt.toISOString(),\n hasRecentFrame: svc.getVisionPipeline().hasRecentFrame(c.id),\n }));\n\n return {\n status: 200,\n body: {\n connected: conns.length > 0,\n connections: conns,\n },\n };\n },\n};\n"],"mappings":"AACA;AAAA,EACE;AAAA,OAEK;AAEA,MAAM,gBAAuB;AAAA,EAClC,MAAM;AAAA,EACN,MAAM;AAAA,EACN,aAAa;AAAA,EACb,cAAc,OAAO,QAAQ;AAC3B,UAAM,MAAM,IAAI,QAAQ,WAA6B,eAAe;AAEpE,QAAI,CAAC,KAAK;AACR,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,MAAM,EAAE,OAAO,yBAAyB;AAAA,MAC1C;AAAA,IACF;AAEA,UAAM,QAAQ,IAAI,eAAe,EAAE,IAAI,CAAC,OAAO;AAAA,MAC7C,IAAI,EAAE;AAAA,MACN,YAAY,EAAE;AAAA,MACd,aAAa,EAAE,YAAY,YAAY;AAAA,MACvC,gBAAgB,IAAI,kBAAkB,EAAE,eAAe,EAAE,EAAE;AAAA,IAC7D,EAAE;AAEF,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,MAAM;AAAA,QACJ,WAAW,MAAM,SAAS;AAAA,QAC1B,aAAa;AAAA,MACf;AAAA,IACF;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Route } from "@elizaos/core";
|
|
2
|
+
/**
|
|
3
|
+
* GET /api/xr/view-host/:id
|
|
4
|
+
*
|
|
5
|
+
* Serves a self-contained HTML page that loads and renders a registered
|
|
6
|
+
* elizaOS view bundle inside an XR-optimised shell. App-xr opens this URL
|
|
7
|
+
* in an iframe that is overlaid on the WebXR scene.
|
|
8
|
+
*
|
|
9
|
+
* The host page:
|
|
10
|
+
* 1. Loads React + ReactDOM from the CDN (same versions as the view bundles).
|
|
11
|
+
* 2. Dynamically imports the view bundle from the agent's /api/views/:id/bundle.js.
|
|
12
|
+
* 3. Mounts the view component inside an XR-friendly dark-theme container.
|
|
13
|
+
* 4. Provides a minimal elizaOS context (agentBaseUrl, viewId, postMessage bridge).
|
|
14
|
+
* 5. Routes voice transcript text to the focused form input.
|
|
15
|
+
*
|
|
16
|
+
* Inter-frame communication (postMessage):
|
|
17
|
+
* Parent → host: { type:"xr:transcript", text:"..." } — fill focused input
|
|
18
|
+
* { type:"xr:focus-next" } — tab to next field
|
|
19
|
+
* Host → parent: { type:"xr:view-ready", viewId:"..." }
|
|
20
|
+
* { type:"xr:navigate", viewId:"..." }
|
|
21
|
+
* { type:"xr:close" }
|
|
22
|
+
*/
|
|
23
|
+
export declare const xrViewHostRoute: Route;
|
|
24
|
+
//# sourceMappingURL=xr-view-host.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-view-host.d.ts","sourceRoot":"","sources":["../../src/routes/xr-view-host.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,eAAe,EAAE,KAkC7B,CAAC"}
|
|
@@ -1,76 +1,35 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* GET /api/xr/view-host/:id
|
|
5
|
-
*
|
|
6
|
-
* Serves a self-contained HTML page that loads and renders a registered
|
|
7
|
-
* elizaOS view bundle inside an XR-optimised shell. App-xr opens this URL
|
|
8
|
-
* in an iframe that is overlaid on the WebXR scene.
|
|
9
|
-
*
|
|
10
|
-
* The host page:
|
|
11
|
-
* 1. Loads React + ReactDOM from the CDN (same versions as the view bundles).
|
|
12
|
-
* 2. Dynamically imports the view bundle from the agent's /api/views/:id/bundle.js.
|
|
13
|
-
* 3. Mounts the view component inside an XR-friendly dark-theme container.
|
|
14
|
-
* 4. Provides a minimal elizaOS context (agentBaseUrl, viewId, postMessage bridge).
|
|
15
|
-
* 5. Routes voice transcript text to the focused form input.
|
|
16
|
-
*
|
|
17
|
-
* Inter-frame communication (postMessage):
|
|
18
|
-
* Parent → host: { type:"xr:transcript", text:"..." } — fill focused input
|
|
19
|
-
* { type:"xr:focus-next" } — tab to next field
|
|
20
|
-
* Host → parent: { type:"xr:view-ready", viewId:"..." }
|
|
21
|
-
* { type:"xr:navigate", viewId:"..." }
|
|
22
|
-
* { type:"xr:close" }
|
|
23
|
-
*/
|
|
24
|
-
export const xrViewHostRoute: Route = {
|
|
1
|
+
const xrViewHostRoute = {
|
|
25
2
|
type: "GET",
|
|
26
3
|
path: "/xr/view-host/:id",
|
|
27
|
-
description:
|
|
28
|
-
"Serves a self-contained XR-friendly HTML host page for a registered view",
|
|
4
|
+
description: "Serves a self-contained XR-friendly HTML host page for a registered view",
|
|
29
5
|
routeHandler: async (ctx) => {
|
|
30
|
-
const viewId =
|
|
6
|
+
const viewId = ctx.params?.id ?? "";
|
|
31
7
|
if (!viewId) {
|
|
32
8
|
return { status: 400, body: { error: "Missing view id" } };
|
|
33
9
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const agentPort = (ctx.runtime as { port?: number }).port ?? 31337;
|
|
37
|
-
const agentOrigin =
|
|
38
|
-
process.env.XR_AGENT_URL ?? `http://localhost:${agentPort}`;
|
|
10
|
+
const agentPort = ctx.runtime.port ?? 31337;
|
|
11
|
+
const agentOrigin = process.env.XR_AGENT_URL ?? `http://localhost:${agentPort}`;
|
|
39
12
|
const bundleUrl = `${agentOrigin}/api/views/${viewId}/bundle.js`;
|
|
40
13
|
const viewsApiUrl = `${agentOrigin}/api/views`;
|
|
41
|
-
|
|
42
14
|
const html = buildHostPage(viewId, bundleUrl, viewsApiUrl, agentOrigin);
|
|
43
|
-
|
|
44
15
|
return {
|
|
45
16
|
status: 200,
|
|
46
17
|
headers: {
|
|
47
18
|
"Content-Type": "text/html; charset=utf-8",
|
|
48
19
|
// Relax CSP for dynamic imports of view bundles (same agent origin only)
|
|
49
|
-
"Content-Security-Policy":
|
|
50
|
-
`default-src 'self' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +
|
|
51
|
-
`script-src 'self' 'unsafe-inline' 'unsafe-eval' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +
|
|
52
|
-
`style-src 'self' 'unsafe-inline'; ` +
|
|
53
|
-
`connect-src 'self' ${agentOrigin} ws://localhost:*;`,
|
|
20
|
+
"Content-Security-Policy": `default-src 'self' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; script-src 'self' 'unsafe-inline' 'unsafe-eval' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; connect-src 'self' ${agentOrigin} ws://localhost:*;`
|
|
54
21
|
},
|
|
55
|
-
body: html
|
|
22
|
+
body: html
|
|
56
23
|
};
|
|
57
|
-
}
|
|
24
|
+
}
|
|
58
25
|
};
|
|
59
|
-
|
|
60
|
-
// ─────────────────────────────────────────────────────────────────────────────
|
|
61
|
-
|
|
62
|
-
function buildHostPage(
|
|
63
|
-
viewId: string,
|
|
64
|
-
bundleUrl: string,
|
|
65
|
-
viewsApiUrl: string,
|
|
66
|
-
agentOrigin: string,
|
|
67
|
-
): string {
|
|
26
|
+
function buildHostPage(viewId, bundleUrl, viewsApiUrl, agentOrigin) {
|
|
68
27
|
return `<!DOCTYPE html>
|
|
69
28
|
<html lang="en" data-view-id="${viewId}">
|
|
70
29
|
<head>
|
|
71
30
|
<meta charset="utf-8" />
|
|
72
31
|
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" />
|
|
73
|
-
<title>XR
|
|
32
|
+
<title>XR \u2013 ${viewId}</title>
|
|
74
33
|
<style>
|
|
75
34
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0 }
|
|
76
35
|
|
|
@@ -216,20 +175,20 @@ function buildHostPage(
|
|
|
216
175
|
<div class="voice-dot"></div>
|
|
217
176
|
<span>Listening</span>
|
|
218
177
|
</div>
|
|
219
|
-
<button class="xr-btn" id="btn-close" title="Close panel"
|
|
178
|
+
<button class="xr-btn" id="btn-close" title="Close panel">\u2715</button>
|
|
220
179
|
</div>
|
|
221
180
|
|
|
222
181
|
<div id="view-mount">
|
|
223
182
|
<div id="view-loader">
|
|
224
183
|
<div class="spinner"></div>
|
|
225
|
-
<span>Loading ${viewId}
|
|
184
|
+
<span>Loading ${viewId}\u2026</span>
|
|
226
185
|
</div>
|
|
227
186
|
</div>
|
|
228
187
|
</div>
|
|
229
188
|
|
|
230
189
|
<div id="transcript-toast"></div>
|
|
231
190
|
|
|
232
|
-
<!-- React from CDN
|
|
191
|
+
<!-- React from CDN \u2014 must match the version view bundles are built against -->
|
|
233
192
|
<script type="importmap">
|
|
234
193
|
{
|
|
235
194
|
"imports": {
|
|
@@ -247,7 +206,7 @@ function buildHostPage(
|
|
|
247
206
|
const VIEWS_API = "${viewsApiUrl}";
|
|
248
207
|
const AGENT_ORIGIN = "${agentOrigin}";
|
|
249
208
|
|
|
250
|
-
//
|
|
209
|
+
// \u2500\u2500 postMessage bridge \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
251
210
|
|
|
252
211
|
function notifyParent(msg) {
|
|
253
212
|
window.parent.postMessage(msg, "*");
|
|
@@ -260,7 +219,7 @@ function buildHostPage(
|
|
|
260
219
|
if (ev.data?.type === "xr:voice-end") showVoiceIndicator(false);
|
|
261
220
|
});
|
|
262
221
|
|
|
263
|
-
//
|
|
222
|
+
// \u2500\u2500 Voice input helpers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
264
223
|
|
|
265
224
|
function fillFocusedInput(text) {
|
|
266
225
|
const el = document.activeElement;
|
|
@@ -300,13 +259,13 @@ function buildHostPage(
|
|
|
300
259
|
toast._t = setTimeout(() => toast.classList.remove("show"), 3000);
|
|
301
260
|
}
|
|
302
261
|
|
|
303
|
-
//
|
|
262
|
+
// \u2500\u2500 Close button \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
304
263
|
|
|
305
264
|
document.getElementById("btn-close")?.addEventListener("click", () => {
|
|
306
265
|
notifyParent({ type: "xr:close", viewId: VIEW_ID });
|
|
307
266
|
});
|
|
308
267
|
|
|
309
|
-
//
|
|
268
|
+
// \u2500\u2500 Mount the view \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
310
269
|
|
|
311
270
|
async function mountView() {
|
|
312
271
|
const mount = document.getElementById("view-mount");
|
|
@@ -344,7 +303,7 @@ function buildHostPage(
|
|
|
344
303
|
console.error("[xr-host] Failed to mount view:", err);
|
|
345
304
|
if (loader) loader.innerHTML =
|
|
346
305
|
\`<div style="color:#f87171;text-align:center;padding:24px">
|
|
347
|
-
<div style="font-size:1.5rem;margin-bottom:8px"
|
|
306
|
+
<div style="font-size:1.5rem;margin-bottom:8px">\u26A0 Load error</div>
|
|
348
307
|
<div style="font-size:0.85rem;color:#a1a1aa">\${err.message}</div>
|
|
349
308
|
<button class="xr-btn" style="margin-top:16px" onclick="mountView()">Retry</button>
|
|
350
309
|
</div>\`;
|
|
@@ -357,3 +316,7 @@ function buildHostPage(
|
|
|
357
316
|
</body>
|
|
358
317
|
</html>`;
|
|
359
318
|
}
|
|
319
|
+
export {
|
|
320
|
+
xrViewHostRoute
|
|
321
|
+
};
|
|
322
|
+
//# sourceMappingURL=xr-view-host.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/routes/xr-view-host.ts"],"sourcesContent":["import type { Route } from \"@elizaos/core\";\n\n/**\n * GET /api/xr/view-host/:id\n *\n * Serves a self-contained HTML page that loads and renders a registered\n * elizaOS view bundle inside an XR-optimised shell. App-xr opens this URL\n * in an iframe that is overlaid on the WebXR scene.\n *\n * The host page:\n * 1. Loads React + ReactDOM from the CDN (same versions as the view bundles).\n * 2. Dynamically imports the view bundle from the agent's /api/views/:id/bundle.js.\n * 3. Mounts the view component inside an XR-friendly dark-theme container.\n * 4. Provides a minimal elizaOS context (agentBaseUrl, viewId, postMessage bridge).\n * 5. Routes voice transcript text to the focused form input.\n *\n * Inter-frame communication (postMessage):\n * Parent → host: { type:\"xr:transcript\", text:\"...\" } — fill focused input\n * { type:\"xr:focus-next\" } — tab to next field\n * Host → parent: { type:\"xr:view-ready\", viewId:\"...\" }\n * { type:\"xr:navigate\", viewId:\"...\" }\n * { type:\"xr:close\" }\n */\nexport const xrViewHostRoute: Route = {\n type: \"GET\",\n path: \"/xr/view-host/:id\",\n description:\n \"Serves a self-contained XR-friendly HTML host page for a registered view\",\n routeHandler: async (ctx) => {\n const viewId = (ctx.params as Record<string, string>)?.id ?? \"\";\n if (!viewId) {\n return { status: 400, body: { error: \"Missing view id\" } };\n }\n\n // Resolve the agent origin so the page can load the bundle\n const agentPort = (ctx.runtime as { port?: number }).port ?? 31337;\n const agentOrigin =\n process.env.XR_AGENT_URL ?? `http://localhost:${agentPort}`;\n const bundleUrl = `${agentOrigin}/api/views/${viewId}/bundle.js`;\n const viewsApiUrl = `${agentOrigin}/api/views`;\n\n const html = buildHostPage(viewId, bundleUrl, viewsApiUrl, agentOrigin);\n\n return {\n status: 200,\n headers: {\n \"Content-Type\": \"text/html; charset=utf-8\",\n // Relax CSP for dynamic imports of view bundles (same agent origin only)\n \"Content-Security-Policy\":\n `default-src 'self' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +\n `script-src 'self' 'unsafe-inline' 'unsafe-eval' ${agentOrigin} https://esm.sh https://cdn.jsdelivr.net; ` +\n `style-src 'self' 'unsafe-inline'; ` +\n `connect-src 'self' ${agentOrigin} ws://localhost:*;`,\n },\n body: html,\n };\n },\n};\n\n// ─────────────────────────────────────────────────────────────────────────────\n\nfunction buildHostPage(\n viewId: string,\n bundleUrl: string,\n viewsApiUrl: string,\n agentOrigin: string,\n): string {\n return `<!DOCTYPE html>\n<html lang=\"en\" data-view-id=\"${viewId}\">\n<head>\n <meta charset=\"utf-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1, viewport-fit=cover\" />\n <title>XR – ${viewId}</title>\n <style>\n *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0 }\n\n :root {\n --bg: #0d0d0f;\n --surface: #18181b;\n --border: rgba(255,255,255,0.08);\n --text: #f4f4f5;\n --muted: #a1a1aa;\n --accent: #6366f1;\n --radius: 12px;\n --font: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, sans-serif;\n }\n\n html, body {\n width: 100%; height: 100%;\n background: var(--bg);\n color: var(--text);\n font-family: var(--font);\n font-size: 18px; /* large for XR readability */\n line-height: 1.5;\n overflow: hidden;\n }\n\n #xr-shell {\n display: flex;\n flex-direction: column;\n height: 100%;\n position: relative;\n }\n\n /* XR header bar */\n #xr-bar {\n display: flex;\n align-items: center;\n gap: 10px;\n padding: 8px 16px;\n background: var(--surface);\n border-bottom: 1px solid var(--border);\n flex-shrink: 0;\n user-select: none;\n }\n #xr-bar-title {\n flex: 1;\n font-weight: 600;\n font-size: 1rem;\n color: var(--text);\n }\n .xr-btn {\n background: var(--border);\n border: none;\n border-radius: 8px;\n color: var(--text);\n cursor: pointer;\n font-size: 1rem;\n padding: 6px 12px;\n transition: background 0.15s;\n }\n .xr-btn:hover { background: rgba(255,255,255,0.15) }\n\n /* Voice indicator */\n #voice-indicator {\n display: none;\n align-items: center;\n gap: 6px;\n padding: 4px 10px;\n border-radius: 20px;\n background: var(--accent);\n font-size: 0.8rem;\n font-weight: 600;\n }\n #voice-indicator.active { display: flex }\n .voice-dot {\n width: 8px; height: 8px;\n border-radius: 50%;\n background: #fff;\n animation: pulse 1s ease-in-out infinite;\n }\n @keyframes pulse { 0%,100% { opacity: 1 } 50% { opacity: 0.4 } }\n\n /* Content area */\n #view-mount {\n flex: 1;\n overflow: auto;\n position: relative;\n }\n\n /* Loading / error states */\n #view-loader {\n display: flex;\n flex-direction: column;\n align-items: center;\n justify-content: center;\n height: 100%;\n gap: 12px;\n color: var(--muted);\n }\n .spinner {\n width: 36px; height: 36px;\n border: 3px solid var(--border);\n border-top-color: var(--accent);\n border-radius: 50%;\n animation: spin 0.8s linear infinite;\n }\n @keyframes spin { to { transform: rotate(360deg) } }\n\n /* XR-friendly form overrides for injected views */\n #view-mount input, #view-mount textarea, #view-mount select {\n font-size: 1rem !important;\n min-height: 44px;\n }\n #view-mount button {\n min-height: 44px;\n min-width: 44px;\n }\n\n /* Transcript toast */\n #transcript-toast {\n position: fixed;\n bottom: 12px; left: 50%;\n transform: translateX(-50%);\n background: rgba(99,102,241,0.9);\n color: #fff;\n border-radius: 20px;\n padding: 6px 16px;\n font-size: 0.85rem;\n pointer-events: none;\n opacity: 0;\n transition: opacity 0.2s;\n white-space: nowrap;\n max-width: 90vw;\n overflow: hidden;\n text-overflow: ellipsis;\n }\n #transcript-toast.show { opacity: 1 }\n </style>\n</head>\n<body>\n <div id=\"xr-shell\">\n <div id=\"xr-bar\">\n <span id=\"xr-bar-title\">${viewId}</span>\n <div id=\"voice-indicator\">\n <div class=\"voice-dot\"></div>\n <span>Listening</span>\n </div>\n <button class=\"xr-btn\" id=\"btn-close\" title=\"Close panel\">✕</button>\n </div>\n\n <div id=\"view-mount\">\n <div id=\"view-loader\">\n <div class=\"spinner\"></div>\n <span>Loading ${viewId}…</span>\n </div>\n </div>\n </div>\n\n <div id=\"transcript-toast\"></div>\n\n <!-- React from CDN — must match the version view bundles are built against -->\n <script type=\"importmap\">\n {\n \"imports\": {\n \"react\": \"https://esm.sh/react@18\",\n \"react-dom\": \"https://esm.sh/react-dom@18\",\n \"react-dom/client\": \"https://esm.sh/react-dom@18/client\",\n \"react/jsx-runtime\": \"https://esm.sh/react@18/jsx-runtime\"\n }\n }\n </script>\n\n <script type=\"module\">\n const VIEW_ID = \"${viewId}\";\n const BUNDLE_URL = \"${bundleUrl}\";\n const VIEWS_API = \"${viewsApiUrl}\";\n const AGENT_ORIGIN = \"${agentOrigin}\";\n\n // ── postMessage bridge ───────────────────────────────────────────────────\n\n function notifyParent(msg) {\n window.parent.postMessage(msg, \"*\");\n }\n\n window.addEventListener(\"message\", (ev) => {\n if (ev.data?.type === \"xr:transcript\") fillFocusedInput(ev.data.text);\n if (ev.data?.type === \"xr:focus-next\") focusNext();\n if (ev.data?.type === \"xr:voice-start\") showVoiceIndicator(true);\n if (ev.data?.type === \"xr:voice-end\") showVoiceIndicator(false);\n });\n\n // ── Voice input helpers ──────────────────────────────────────────────────\n\n function fillFocusedInput(text) {\n const el = document.activeElement;\n if (!el) return;\n const tag = el.tagName;\n if (tag === \"INPUT\" || tag === \"TEXTAREA\" || tag === \"SELECT\") {\n const native = Object.getOwnPropertyDescriptor(window[tag === \"INPUT\" ? \"HTMLInputElement\" : tag === \"TEXTAREA\" ? \"HTMLTextAreaElement\" : \"HTMLSelectElement\"].prototype, \"value\");\n native.set.call(el, text);\n el.dispatchEvent(new Event(\"input\", { bubbles: true }));\n el.dispatchEvent(new Event(\"change\", { bubbles: true }));\n showTranscript(text);\n } else if (el.getAttribute(\"role\") === \"combobox\" || el.getAttribute(\"role\") === \"listbox\") {\n el.dispatchEvent(new CustomEvent(\"xr:voice-select\", { bubbles: true, detail: { text } }));\n showTranscript(text);\n }\n }\n\n function focusNext() {\n const all = Array.from(document.querySelectorAll(\"input,textarea,button,select,[tabindex]\"))\n .filter(el => !el.disabled && !el.closest(\"[hidden]\"));\n const idx = all.indexOf(document.activeElement);\n const next = all[idx + 1] ?? all[0];\n next?.focus();\n }\n\n function showVoiceIndicator(active) {\n const el = document.getElementById(\"voice-indicator\");\n if (el) el.classList.toggle(\"active\", active);\n }\n\n function showTranscript(text) {\n const toast = document.getElementById(\"transcript-toast\");\n if (!toast) return;\n toast.textContent = text;\n toast.classList.add(\"show\");\n clearTimeout(toast._t);\n toast._t = setTimeout(() => toast.classList.remove(\"show\"), 3000);\n }\n\n // ── Close button ─────────────────────────────────────────────────────────\n\n document.getElementById(\"btn-close\")?.addEventListener(\"click\", () => {\n notifyParent({ type: \"xr:close\", viewId: VIEW_ID });\n });\n\n // ── Mount the view ───────────────────────────────────────────────────────\n\n async function mountView() {\n const mount = document.getElementById(\"view-mount\");\n const loader = document.getElementById(\"view-loader\");\n\n try {\n // Provide minimal elizaOS-like context so views can render\n window.__elizaXRContext = {\n agentBaseUrl: AGENT_ORIGIN,\n viewId: VIEW_ID,\n fetchViews: () => fetch(VIEWS_API).then(r => r.json()),\n navigate: (id) => notifyParent({ type: \"xr:navigate\", viewId: id }),\n };\n\n const mod = await import(/* @vite-ignore */ BUNDLE_URL);\n const component = mod.default ?? mod[Object.keys(mod)[0]];\n\n if (!component) throw new Error(\"No component export found in bundle\");\n\n // Dynamically import React + ReactDOM\n const [React, ReactDOMClient] = await Promise.all([\n import(\"react\"),\n import(\"react-dom/client\"),\n ]);\n\n if (loader) loader.style.display = \"none\";\n\n // Render the view component\n const root = ReactDOMClient.createRoot(mount);\n root.render(React.createElement(component));\n\n notifyParent({ type: \"xr:view-ready\", viewId: VIEW_ID });\n\n } catch (err) {\n console.error(\"[xr-host] Failed to mount view:\", err);\n if (loader) loader.innerHTML =\n \\`<div style=\"color:#f87171;text-align:center;padding:24px\">\n <div style=\"font-size:1.5rem;margin-bottom:8px\">⚠ Load error</div>\n <div style=\"font-size:0.85rem;color:#a1a1aa\">\\${err.message}</div>\n <button class=\"xr-btn\" style=\"margin-top:16px\" onclick=\"mountView()\">Retry</button>\n </div>\\`;\n notifyParent({ type: \"xr:view-error\", viewId: VIEW_ID, error: err.message });\n }\n }\n\n mountView();\n </script>\n</body>\n</html>`;\n}\n"],"mappings":"AAuBO,MAAM,kBAAyB;AAAA,EACpC,MAAM;AAAA,EACN,MAAM;AAAA,EACN,aACE;AAAA,EACF,cAAc,OAAO,QAAQ;AAC3B,UAAM,SAAU,IAAI,QAAmC,MAAM;AAC7D,QAAI,CAAC,QAAQ;AACX,aAAO,EAAE,QAAQ,KAAK,MAAM,EAAE,OAAO,kBAAkB,EAAE;AAAA,IAC3D;AAGA,UAAM,YAAa,IAAI,QAA8B,QAAQ;AAC7D,UAAM,cACJ,QAAQ,IAAI,gBAAgB,oBAAoB,SAAS;AAC3D,UAAM,YAAY,GAAG,WAAW,cAAc,MAAM;AACpD,UAAM,cAAc,GAAG,WAAW;AAElC,UAAM,OAAO,cAAc,QAAQ,WAAW,aAAa,WAAW;AAEtE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,gBAAgB;AAAA;AAAA,QAEhB,2BACE,sBAAsB,WAAW,6FACkB,WAAW,kGAExC,WAAW;AAAA,MACrC;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AACF;AAIA,SAAS,cACP,QACA,WACA,aACA,aACQ;AACR,SAAO;AAAA,gCACuB,MAAM;AAAA;AAAA;AAAA;AAAA,qBAItB,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gCA6IU,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,wBAWd,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAoBP,MAAM;AAAA,0BACH,SAAS;AAAA,yBACV,WAAW;AAAA,4BACR,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+GvC;","names":[]}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Route } from "@elizaos/core";
|
|
2
|
+
/**
|
|
3
|
+
* GET /api/xr/views
|
|
4
|
+
* Returns all XR-typed views from registered plugins.
|
|
5
|
+
* Used by app-xr to populate the view launcher.
|
|
6
|
+
*/
|
|
7
|
+
export declare const xrViewsRoute: Route;
|
|
8
|
+
//# sourceMappingURL=xr-views.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-views.d.ts","sourceRoot":"","sources":["../../src/routes/xr-views.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAM3C;;;;GAIG;AACH,eAAO,MAAM,YAAY,EAAE,KA8B1B,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { listViews } from "@elizaos/agent/api/views-registry";
|
|
2
|
+
import {
|
|
3
|
+
XR_SERVICE_TYPE
|
|
4
|
+
} from "../services/xr-session-service.js";
|
|
5
|
+
const xrViewsRoute = {
|
|
6
|
+
type: "GET",
|
|
7
|
+
path: "/xr/views",
|
|
8
|
+
description: "Lists all XR-capable views from registered plugins",
|
|
9
|
+
routeHandler: async (ctx) => {
|
|
10
|
+
const views = listViews({ developerMode: true, viewType: "xr" }).filter((v) => v.viewType === "xr").map((v) => ({
|
|
11
|
+
id: v.id,
|
|
12
|
+
label: v.label,
|
|
13
|
+
icon: v.icon,
|
|
14
|
+
description: v.description,
|
|
15
|
+
tags: v.tags,
|
|
16
|
+
xrOptions: v.xrOptions,
|
|
17
|
+
path: v.path,
|
|
18
|
+
pluginName: v.pluginName,
|
|
19
|
+
available: v.available
|
|
20
|
+
}));
|
|
21
|
+
const connections = ctx.runtime.getService(XR_SERVICE_TYPE)?.getConnections().map((c) => ({ id: c.id, deviceType: c.deviceType })) ?? [];
|
|
22
|
+
return {
|
|
23
|
+
status: 200,
|
|
24
|
+
body: { views, connections, count: views.length }
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
export {
|
|
29
|
+
xrViewsRoute
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=xr-views.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/routes/xr-views.ts"],"sourcesContent":["import { listViews } from \"@elizaos/agent/api/views-registry\";\nimport type { Route } from \"@elizaos/core\";\nimport {\n XR_SERVICE_TYPE,\n type XRSessionService,\n} from \"../services/xr-session-service.js\";\n\n/**\n * GET /api/xr/views\n * Returns all XR-typed views from registered plugins.\n * Used by app-xr to populate the view launcher.\n */\nexport const xrViewsRoute: Route = {\n type: \"GET\",\n path: \"/xr/views\",\n description: \"Lists all XR-capable views from registered plugins\",\n routeHandler: async (ctx) => {\n const views = listViews({ developerMode: true, viewType: \"xr\" })\n .filter((v) => v.viewType === \"xr\")\n .map((v) => ({\n id: v.id,\n label: v.label,\n icon: v.icon,\n description: v.description,\n tags: v.tags,\n xrOptions: v.xrOptions,\n path: v.path,\n pluginName: v.pluginName,\n available: v.available,\n }));\n\n const connections =\n ctx.runtime\n .getService<XRSessionService>(XR_SERVICE_TYPE)\n ?.getConnections()\n .map((c) => ({ id: c.id, deviceType: c.deviceType })) ?? [];\n\n return {\n status: 200,\n body: { views, connections, count: views.length },\n };\n },\n};\n"],"mappings":"AAAA,SAAS,iBAAiB;AAE1B;AAAA,EACE;AAAA,OAEK;AAOA,MAAM,eAAsB;AAAA,EACjC,MAAM;AAAA,EACN,MAAM;AAAA,EACN,aAAa;AAAA,EACb,cAAc,OAAO,QAAQ;AAC3B,UAAM,QAAQ,UAAU,EAAE,eAAe,MAAM,UAAU,KAAK,CAAC,EAC5D,OAAO,CAAC,MAAM,EAAE,aAAa,IAAI,EACjC,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,MAAM,EAAE;AAAA,MACR,aAAa,EAAE;AAAA,MACf,MAAM,EAAE;AAAA,MACR,WAAW,EAAE;AAAA,MACb,MAAM,EAAE;AAAA,MACR,YAAY,EAAE;AAAA,MACd,WAAW,EAAE;AAAA,IACf,EAAE;AAEJ,UAAM,cACJ,IAAI,QACD,WAA6B,eAAe,GAC3C,eAAe,EAChB,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC;AAE9D,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,MAAM,EAAE,OAAO,aAAa,OAAO,MAAM,OAAO;AAAA,IAClD;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { IAgentRuntime } from "@elizaos/core";
|
|
2
|
+
import type { XRAudioHeader } from "../protocol.ts";
|
|
3
|
+
export interface PendingTranscription {
|
|
4
|
+
chunks: Buffer[];
|
|
5
|
+
firstTs: number;
|
|
6
|
+
lastTs: number;
|
|
7
|
+
encoding: XRAudioHeader["encoding"];
|
|
8
|
+
sampleRate: number;
|
|
9
|
+
silenceTimer?: ReturnType<typeof setTimeout>;
|
|
10
|
+
}
|
|
11
|
+
export declare class AudioPipeline {
|
|
12
|
+
private readonly runtime;
|
|
13
|
+
private readonly onTranscript;
|
|
14
|
+
private pending;
|
|
15
|
+
constructor(runtime: IAgentRuntime, onTranscript: (connectionId: string, text: string) => Promise<void>);
|
|
16
|
+
push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void;
|
|
17
|
+
flush(connectionId: string): Promise<void>;
|
|
18
|
+
clear(connectionId: string): void;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=audio-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audio-pipeline.d.ts","sourceRoot":"","sources":["../../src/services/audio-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA4BpD,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;CAC9C;AAED,qBAAa,aAAa;IAItB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAJ/B,OAAO,CAAC,OAAO,CAA2C;gBAGvC,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,CAC7B,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,KACT,OAAO,CAAC,IAAI,CAAC;IAGpB,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IA6BhE,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmChD,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;CAKlC"}
|