@elizaos/plugin-xr 2.0.3-beta.5 → 2.0.3-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/actions/xr-query-vision.d.ts +3 -0
  2. package/dist/actions/xr-query-vision.d.ts.map +1 -0
  3. package/dist/actions/xr-query-vision.js +39 -0
  4. package/dist/actions/xr-query-vision.js.map +1 -0
  5. package/dist/actions/xr-view-actions.d.ts +18 -0
  6. package/dist/actions/xr-view-actions.d.ts.map +1 -0
  7. package/dist/actions/xr-view-actions.js +304 -0
  8. package/dist/actions/xr-view-actions.js.map +1 -0
  9. package/dist/index.d.ts +8 -0
  10. package/dist/index.d.ts.map +1 -0
  11. package/dist/index.js +57 -0
  12. package/dist/index.js.map +1 -0
  13. package/dist/protocol.d.ts +124 -0
  14. package/dist/protocol.d.ts.map +1 -0
  15. package/dist/protocol.js +18 -0
  16. package/dist/protocol.js.map +1 -0
  17. package/dist/providers/xr-context.d.ts +3 -0
  18. package/dist/providers/xr-context.d.ts.map +1 -0
  19. package/dist/providers/xr-context.js +34 -0
  20. package/dist/providers/xr-context.js.map +1 -0
  21. package/dist/routes/xr-connect.d.ts +3 -0
  22. package/dist/routes/xr-connect.d.ts.map +1 -0
  23. package/{src/routes/xr-connect.ts → dist/routes/xr-connect.js} +12 -15
  24. package/dist/routes/xr-connect.js.map +1 -0
  25. package/dist/routes/xr-simulator-route.d.ts +8 -0
  26. package/dist/routes/xr-simulator-route.d.ts.map +1 -0
  27. package/{src/routes/xr-simulator-route.ts → dist/routes/xr-simulator-route.js} +10 -16
  28. package/dist/routes/xr-simulator-route.js.map +1 -0
  29. package/dist/routes/xr-status.d.ts +3 -0
  30. package/dist/routes/xr-status.d.ts.map +1 -0
  31. package/{src/routes/xr-status.ts → dist/routes/xr-status.js} +13 -15
  32. package/dist/routes/xr-status.js.map +1 -0
  33. package/dist/routes/xr-view-host.d.ts +24 -0
  34. package/dist/routes/xr-view-host.d.ts.map +1 -0
  35. package/{src/routes/xr-view-host.ts → dist/routes/xr-view-host.js} +22 -59
  36. package/dist/routes/xr-view-host.js.map +1 -0
  37. package/dist/routes/xr-views.d.ts +8 -0
  38. package/dist/routes/xr-views.d.ts.map +1 -0
  39. package/dist/routes/xr-views.js +31 -0
  40. package/dist/routes/xr-views.js.map +1 -0
  41. package/dist/services/audio-pipeline.d.ts +20 -0
  42. package/dist/services/audio-pipeline.d.ts.map +1 -0
  43. package/{src/services/audio-pipeline.ts → dist/services/audio-pipeline.js} +25 -58
  44. package/dist/services/audio-pipeline.js.map +1 -0
  45. package/dist/services/vision-pipeline.d.ts +16 -0
  46. package/dist/services/vision-pipeline.d.ts.map +1 -0
  47. package/dist/services/vision-pipeline.js +39 -0
  48. package/dist/services/vision-pipeline.js.map +1 -0
  49. package/dist/services/xr-session-service.d.ts +50 -0
  50. package/dist/services/xr-session-service.d.ts.map +1 -0
  51. package/{src/services/xr-session-service.ts → dist/services/xr-session-service.js} +85 -194
  52. package/dist/services/xr-session-service.js.map +1 -0
  53. package/package.json +9 -4
  54. package/AGENTS.md +0 -151
  55. package/CLAUDE.md +0 -151
  56. package/simulator/bun.lock +0 -159
  57. package/simulator/package.json +0 -28
  58. package/simulator/src/emulator.ts +0 -174
  59. package/simulator/src/mock-agent.ts +0 -233
  60. package/simulator/src/node.ts +0 -9
  61. package/simulator/src/playwright-fixture.ts +0 -169
  62. package/simulator/src/types.ts +0 -51
  63. package/simulator/tsconfig.json +0 -13
  64. package/simulator/vite.config.ts +0 -25
  65. package/src/__tests__/audio-pipeline.test.ts +0 -129
  66. package/src/__tests__/protocol.test.ts +0 -53
  67. package/src/__tests__/routes-e2e.test.ts +0 -276
  68. package/src/__tests__/vision-pipeline.test.ts +0 -73
  69. package/src/__tests__/xr-bundle-coverage.test.ts +0 -303
  70. package/src/__tests__/xr-feature-parity.test.ts +0 -524
  71. package/src/__tests__/xr-functional-parity.test.ts +0 -522
  72. package/src/__tests__/xr-view-host-http.test.ts +0 -239
  73. package/src/__tests__/xr-view-host.test.ts +0 -174
  74. package/src/actions/xr-query-vision.ts +0 -64
  75. package/src/actions/xr-view-actions.ts +0 -386
  76. package/src/index.ts +0 -55
  77. package/src/protocol.ts +0 -126
  78. package/src/providers/xr-context.ts +0 -49
  79. package/src/routes/xr-views.ts +0 -43
  80. package/src/services/vision-pipeline.ts +0 -57
  81. package/tsconfig.build.json +0 -9
  82. package/tsconfig.json +0 -30
  83. package/vitest.config.ts +0 -21
@@ -1,10 +1,6 @@
1
- import type { IAgentRuntime } from "@elizaos/core";
2
1
  import { ModelType } from "@elizaos/core";
3
- import type { XRAudioHeader } from "../protocol.ts";
4
-
5
- // Prepend a RIFF/WAV header so Whisper can decode raw Float32 PCM.
6
- function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
7
- const channels = 1; // ScriptProcessorNode fallback is always mono
2
+ function pcmF32ToWav(pcmData, sampleRate) {
3
+ const channels = 1;
8
4
  const dataSize = pcmData.length;
9
5
  const header = Buffer.alloc(44);
10
6
  header.write("RIFF", 0);
@@ -12,7 +8,7 @@ function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
12
8
  header.write("WAVE", 8);
13
9
  header.write("fmt ", 12);
14
10
  header.writeUInt32LE(16, 16);
15
- header.writeUInt16LE(3, 20); // IEEE_FLOAT
11
+ header.writeUInt16LE(3, 20);
16
12
  header.writeUInt16LE(channels, 22);
17
13
  header.writeUInt32LE(sampleRate, 24);
18
14
  header.writeUInt32LE(sampleRate * channels * 4, 28);
@@ -22,33 +18,17 @@ function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
22
18
  header.writeUInt32LE(dataSize, 40);
23
19
  return Buffer.concat([header, pcmData]);
24
20
  }
25
-
26
- // Accumulate up to FLUSH_AFTER_MS of audio then transcribe.
27
- // Also flush if no chunk arrives within SILENCE_GAP_MS (end-of-utterance).
28
- const FLUSH_AFTER_MS = 2000;
21
+ const FLUSH_AFTER_MS = 2e3;
29
22
  const SILENCE_GAP_MS = 1500;
30
-
31
- export interface PendingTranscription {
32
- chunks: Buffer[];
33
- firstTs: number;
34
- lastTs: number;
35
- encoding: XRAudioHeader["encoding"];
36
- sampleRate: number;
37
- silenceTimer?: ReturnType<typeof setTimeout>;
38
- }
39
-
40
- export class AudioPipeline {
41
- private pending = new Map<string, PendingTranscription>();
42
-
43
- constructor(
44
- private readonly runtime: IAgentRuntime,
45
- private readonly onTranscript: (
46
- connectionId: string,
47
- text: string,
48
- ) => Promise<void>,
49
- ) {}
50
-
51
- push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void {
23
+ class AudioPipeline {
24
+ constructor(runtime, onTranscript) {
25
+ this.runtime = runtime;
26
+ this.onTranscript = onTranscript;
27
+ }
28
+ runtime;
29
+ onTranscript;
30
+ pending = /* @__PURE__ */ new Map();
31
+ push(connectionId, header, chunk) {
52
32
  let state = this.pending.get(connectionId);
53
33
  if (!state) {
54
34
  state = {
@@ -56,65 +36,52 @@ export class AudioPipeline {
56
36
  firstTs: header.ts,
57
37
  lastTs: header.ts,
58
38
  encoding: header.encoding,
59
- sampleRate: header.sampleRate,
39
+ sampleRate: header.sampleRate
60
40
  };
61
41
  this.pending.set(connectionId, state);
62
42
  }
63
-
64
43
  state.chunks.push(chunk);
65
44
  state.lastTs = header.ts;
66
-
67
- // Reset silence timer on each incoming chunk
68
45
  if (state.silenceTimer) clearTimeout(state.silenceTimer);
69
46
  state.silenceTimer = setTimeout(
70
47
  () => void this.flush(connectionId),
71
- SILENCE_GAP_MS,
48
+ SILENCE_GAP_MS
72
49
  );
73
-
74
- // Also flush if we've accumulated enough audio
75
50
  if (state.lastTs - state.firstTs >= FLUSH_AFTER_MS) {
76
51
  void this.flush(connectionId);
77
52
  }
78
53
  }
79
-
80
- async flush(connectionId: string): Promise<void> {
54
+ async flush(connectionId) {
81
55
  const state = this.pending.get(connectionId);
82
56
  if (!state || state.chunks.length === 0) return;
83
-
84
57
  if (state.silenceTimer) {
85
58
  clearTimeout(state.silenceTimer);
86
- state.silenceTimer = undefined;
59
+ state.silenceTimer = void 0;
87
60
  }
88
61
  this.pending.delete(connectionId);
89
-
90
62
  const combined = Buffer.concat(state.chunks);
91
- if (combined.length < 512) return; // too small to be real speech
92
-
93
- // pcm-f32 is raw Float32 samples (mono, from ScriptProcessorNode fallback).
94
- // Whisper expects a valid audio container — wrap with a WAV header.
95
- const audioBuffer =
96
- state.encoding === "pcm-f32"
97
- ? pcmF32ToWav(combined, state.sampleRate)
98
- : combined;
99
-
63
+ if (combined.length < 512) return;
64
+ const audioBuffer = state.encoding === "pcm-f32" ? pcmF32ToWav(combined, state.sampleRate) : combined;
100
65
  try {
101
66
  const transcript = await this.runtime.useModel(
102
67
  ModelType.TRANSCRIPTION,
103
- audioBuffer,
68
+ audioBuffer
104
69
  );
105
70
  const text = typeof transcript === "string" ? transcript.trim() : "";
106
71
  if (text.length > 0) {
107
72
  await this.onTranscript(connectionId, text);
108
73
  }
109
74
  } catch (err) {
110
- // log but don't crash the pipeline
111
75
  console.error("[plugin-xr] transcription error:", err);
112
76
  }
113
77
  }
114
-
115
- clear(connectionId: string): void {
78
+ clear(connectionId) {
116
79
  const state = this.pending.get(connectionId);
117
80
  if (state?.silenceTimer) clearTimeout(state.silenceTimer);
118
81
  this.pending.delete(connectionId);
119
82
  }
120
83
  }
84
+ export {
85
+ AudioPipeline
86
+ };
87
+ //# sourceMappingURL=audio-pipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/services/audio-pipeline.ts"],"sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { ModelType } from \"@elizaos/core\";\nimport type { XRAudioHeader } from \"../protocol.js\";\n\n// Prepend a RIFF/WAV header so Whisper can decode raw Float32 PCM.\nfunction pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {\n const channels = 1; // ScriptProcessorNode fallback is always mono\n const dataSize = pcmData.length;\n const header = Buffer.alloc(44);\n header.write(\"RIFF\", 0);\n header.writeUInt32LE(36 + dataSize, 4);\n header.write(\"WAVE\", 8);\n header.write(\"fmt \", 12);\n header.writeUInt32LE(16, 16);\n header.writeUInt16LE(3, 20); // IEEE_FLOAT\n header.writeUInt16LE(channels, 22);\n header.writeUInt32LE(sampleRate, 24);\n header.writeUInt32LE(sampleRate * channels * 4, 28);\n header.writeUInt16LE(channels * 4, 32);\n header.writeUInt16LE(32, 34);\n header.write(\"data\", 36);\n header.writeUInt32LE(dataSize, 40);\n return Buffer.concat([header, pcmData]);\n}\n\n// Accumulate up to FLUSH_AFTER_MS of audio then transcribe.\n// Also flush if no chunk arrives within SILENCE_GAP_MS (end-of-utterance).\nconst FLUSH_AFTER_MS = 2000;\nconst SILENCE_GAP_MS = 1500;\n\nexport interface PendingTranscription {\n chunks: Buffer[];\n firstTs: number;\n lastTs: number;\n encoding: XRAudioHeader[\"encoding\"];\n sampleRate: number;\n silenceTimer?: ReturnType<typeof setTimeout>;\n}\n\nexport class AudioPipeline {\n private pending = new Map<string, PendingTranscription>();\n\n constructor(\n private readonly runtime: IAgentRuntime,\n private readonly onTranscript: (\n connectionId: string,\n text: string,\n ) => Promise<void>,\n ) {}\n\n push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void {\n let state = this.pending.get(connectionId);\n if (!state) {\n state = {\n chunks: [],\n firstTs: header.ts,\n lastTs: header.ts,\n encoding: header.encoding,\n sampleRate: header.sampleRate,\n };\n this.pending.set(connectionId, state);\n }\n\n state.chunks.push(chunk);\n state.lastTs = header.ts;\n\n // Reset silence timer on each incoming chunk\n if (state.silenceTimer) clearTimeout(state.silenceTimer);\n state.silenceTimer = setTimeout(\n () => void this.flush(connectionId),\n SILENCE_GAP_MS,\n );\n\n // Also flush if we've accumulated enough audio\n if (state.lastTs - state.firstTs >= FLUSH_AFTER_MS) {\n void this.flush(connectionId);\n }\n }\n\n async flush(connectionId: string): Promise<void> {\n const state = this.pending.get(connectionId);\n if (!state || state.chunks.length === 0) return;\n\n if (state.silenceTimer) {\n clearTimeout(state.silenceTimer);\n state.silenceTimer = undefined;\n }\n this.pending.delete(connectionId);\n\n const combined = Buffer.concat(state.chunks);\n if (combined.length < 512) return; // too small to be real speech\n\n // pcm-f32 is raw Float32 samples (mono, from ScriptProcessorNode fallback).\n // Whisper expects a valid audio container — wrap with a WAV header.\n const audioBuffer =\n state.encoding === \"pcm-f32\"\n ? pcmF32ToWav(combined, state.sampleRate)\n : combined;\n\n try {\n const transcript = await this.runtime.useModel(\n ModelType.TRANSCRIPTION,\n audioBuffer,\n );\n const text = typeof transcript === \"string\" ? transcript.trim() : \"\";\n if (text.length > 0) {\n await this.onTranscript(connectionId, text);\n }\n } catch (err) {\n // log but don't crash the pipeline\n console.error(\"[plugin-xr] transcription error:\", err);\n }\n }\n\n clear(connectionId: string): void {\n const state = this.pending.get(connectionId);\n if (state?.silenceTimer) clearTimeout(state.silenceTimer);\n this.pending.delete(connectionId);\n }\n}\n"],"mappings":"AACA,SAAS,iBAAiB;AAI1B,SAAS,YAAY,SAAiB,YAA4B;AAChE,QAAM,WAAW;AACjB,QAAM,WAAW,QAAQ;AACzB,QAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,SAAO,MAAM,QAAQ,CAAC;AACtB,SAAO,cAAc,KAAK,UAAU,CAAC;AACrC,SAAO,MAAM,QAAQ,CAAC;AACtB,SAAO,MAAM,QAAQ,EAAE;AACvB,SAAO,cAAc,IAAI,EAAE;AAC3B,SAAO,cAAc,GAAG,EAAE;AAC1B,SAAO,cAAc,UAAU,EAAE;AACjC,SAAO,cAAc,YAAY,EAAE;AACnC,SAAO,cAAc,aAAa,WAAW,GAAG,EAAE;AAClD,SAAO,cAAc,WAAW,GAAG,EAAE;AACrC,SAAO,cAAc,IAAI,EAAE;AAC3B,SAAO,MAAM,QAAQ,EAAE;AACvB,SAAO,cAAc,UAAU,EAAE;AACjC,SAAO,OAAO,OAAO,CAAC,QAAQ,OAAO,CAAC;AACxC;AAIA,MAAM,iBAAiB;AACvB,MAAM,iBAAiB;AAWhB,MAAM,cAAc;AAAA,EAGzB,YACmB,SACA,cAIjB;AALiB;AACA;AAAA,EAIhB;AAAA,EALgB;AAAA,EACA;AAAA,EAJX,UAAU,oBAAI,IAAkC;AAAA,EAUxD,KAAK,cAAsB,QAAuB,OAAqB;AACrE,QAAI,QAAQ,KAAK,QAAQ,IAAI,YAAY;AACzC,QAAI,CAAC,OAAO;AACV,cAAQ;AAAA,QACN,QAAQ,CAAC;AAAA,QACT,SAAS,OAAO;AAAA,QAChB,QAAQ,OAAO;AAAA,QACf,UAAU,OAAO;AAAA,QACjB,YAAY,OAAO;AAAA,MACrB;AACA,WAAK,QAAQ,IAAI,cAAc,KAAK;AAAA,IACtC;AAEA,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,SAAS,OAAO;AAGtB,QAAI,MAAM,aAAc,cAAa,MAAM,YAAY;AACvD,UAAM,eAAe;AAAA,MACnB,MAAM,KAAK,KAAK,MAAM,YAAY;AAAA,MAClC;AAAA,IACF;AAGA,QAAI,MAAM,SAAS,MAAM,WAAW,gBAAgB;AAClD,WAAK,KAAK,MAAM,YAAY;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,cAAqC;AAC/C,UAAM,QAAQ,KAAK,QAAQ,IAAI,YAAY;AAC3C,QAAI,CAAC,SAAS,MAAM,OAAO,WAAW,EAAG;AAEzC,QAAI,MAAM,cAAc;AACtB,mBAAa,MAAM,YAAY;AAC/B,YAAM,eAAe;AAAA,IACvB;AACA,SAAK,QAAQ,OAAO,YAAY;AAEhC,UAAM,WAAW,OAAO,OAAO,MAAM,MAAM;AAC3C,QAAI,SAAS,SAAS,IAAK;AAI3B,UAAM,cACJ,MAAM,aAAa,YACf,YAAY,UAAU,MAAM,UAAU,IACtC;AAEN,QAAI;AACF,YAAM,aAAa,MAAM,KAAK,QAAQ;AAAA,QACpC,UAAU;AAAA,QACV;AAAA,MACF;AACA,YAAM,OAAO,OAAO,eAAe,WAAW,WAAW,KAAK,IAAI;AAClE,UAAI,KAAK,SAAS,GAAG;AACnB,cAAM,KAAK,aAAa,cAAc,IAAI;AAAA,MAC5C;AAAA,IACF,SAAS,KAAK;AAEZ,cAAQ,MAAM,oCAAoC,GAAG;AAAA,IACvD;AAAA,EACF;AAAA,EAEA,MAAM,cAA4B;AAChC,UAAM,QAAQ,KAAK,QAAQ,IAAI,YAAY;AAC3C,QAAI,OAAO,aAAc,cAAa,MAAM,YAAY;AACxD,SAAK,QAAQ,OAAO,YAAY;AAAA,EAClC;AACF;","names":[]}
@@ -0,0 +1,16 @@
1
+ import type { IAgentRuntime } from "@elizaos/core";
2
+ import type { XRFrameHeader } from "../protocol.ts";
3
+ export interface LatestFrame {
4
+ data: Buffer;
5
+ header: XRFrameHeader;
6
+ receivedAt: number;
7
+ }
8
+ export declare class VisionPipeline {
9
+ private latest;
10
+ storeFrame(connectionId: string, header: XRFrameHeader, data: Buffer): void;
11
+ getLatestFrame(connectionId: string): LatestFrame | undefined;
12
+ hasRecentFrame(connectionId: string): boolean;
13
+ describeFrame(runtime: IAgentRuntime, connectionId: string, prompt?: string): Promise<string | null>;
14
+ clear(connectionId: string): void;
15
+ }
16
+ //# sourceMappingURL=vision-pipeline.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vision-pipeline.d.ts","sourceRoot":"","sources":["../../src/services/vision-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAkC;IAEhD,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAI3E,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAO7D,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIvC,aAAa,CACjB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAkBzB,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;CAGlC"}
@@ -0,0 +1,39 @@
1
+ import { ModelType } from "@elizaos/core";
2
+ const FRAME_MAX_AGE_MS = 1e4;
3
+ class VisionPipeline {
4
+ latest = /* @__PURE__ */ new Map();
5
+ storeFrame(connectionId, header, data) {
6
+ this.latest.set(connectionId, { data, header, receivedAt: Date.now() });
7
+ }
8
+ getLatestFrame(connectionId) {
9
+ const frame = this.latest.get(connectionId);
10
+ if (!frame) return void 0;
11
+ if (Date.now() - frame.receivedAt > FRAME_MAX_AGE_MS) return void 0;
12
+ return frame;
13
+ }
14
+ hasRecentFrame(connectionId) {
15
+ return this.getLatestFrame(connectionId) !== void 0;
16
+ }
17
+ async describeFrame(runtime, connectionId, prompt) {
18
+ const frame = this.getLatestFrame(connectionId);
19
+ if (!frame) return null;
20
+ const dataUrl = `data:image/${frame.header.format};base64,${frame.data.toString("base64")}`;
21
+ try {
22
+ const description = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, {
23
+ imageUrl: dataUrl,
24
+ prompt: prompt ?? "Describe what you see in this image concisely."
25
+ });
26
+ return typeof description === "string" ? description : null;
27
+ } catch (err) {
28
+ console.error("[plugin-xr] vision error:", err);
29
+ return null;
30
+ }
31
+ }
32
+ clear(connectionId) {
33
+ this.latest.delete(connectionId);
34
+ }
35
+ }
36
+ export {
37
+ VisionPipeline
38
+ };
39
+ //# sourceMappingURL=vision-pipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../../src/services/vision-pipeline.ts"],"sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { ModelType } from \"@elizaos/core\";\nimport type { XRFrameHeader } from \"../protocol.js\";\n\nexport interface LatestFrame {\n data: Buffer;\n header: XRFrameHeader;\n receivedAt: number;\n}\n\n// A frame older than this is considered stale and won't be described\nconst FRAME_MAX_AGE_MS = 10_000;\n\nexport class VisionPipeline {\n private latest = new Map<string, LatestFrame>();\n\n storeFrame(connectionId: string, header: XRFrameHeader, data: Buffer): void {\n this.latest.set(connectionId, { data, header, receivedAt: Date.now() });\n }\n\n getLatestFrame(connectionId: string): LatestFrame | undefined {\n const frame = this.latest.get(connectionId);\n if (!frame) return undefined;\n if (Date.now() - frame.receivedAt > FRAME_MAX_AGE_MS) return undefined;\n return frame;\n }\n\n hasRecentFrame(connectionId: string): boolean {\n return this.getLatestFrame(connectionId) !== undefined;\n }\n\n async describeFrame(\n runtime: IAgentRuntime,\n connectionId: string,\n prompt?: string,\n ): Promise<string | null> {\n const frame = this.getLatestFrame(connectionId);\n if (!frame) return null;\n\n const dataUrl = `data:image/${frame.header.format};base64,${frame.data.toString(\"base64\")}`;\n\n try {\n const description = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, {\n imageUrl: dataUrl,\n prompt: prompt ?? \"Describe what you see in this image concisely.\",\n });\n return typeof description === \"string\" ? description : null;\n } catch (err) {\n console.error(\"[plugin-xr] vision error:\", err);\n return null;\n }\n }\n\n clear(connectionId: string): void {\n this.latest.delete(connectionId);\n }\n}\n"],"mappings":"AACA,SAAS,iBAAiB;AAU1B,MAAM,mBAAmB;AAElB,MAAM,eAAe;AAAA,EAClB,SAAS,oBAAI,IAAyB;AAAA,EAE9C,WAAW,cAAsB,QAAuB,MAAoB;AAC1E,SAAK,OAAO,IAAI,cAAc,EAAE,MAAM,QAAQ,YAAY,KAAK,IAAI,EAAE,CAAC;AAAA,EACxE;AAAA,EAEA,eAAe,cAA+C;AAC5D,UAAM,QAAQ,KAAK,OAAO,IAAI,YAAY;AAC1C,QAAI,CAAC,MAAO,QAAO;AACnB,QAAI,KAAK,IAAI,IAAI,MAAM,aAAa,iBAAkB,QAAO;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,cAA+B;AAC5C,WAAO,KAAK,eAAe,YAAY,MAAM;AAAA,EAC/C;AAAA,EAEA,MAAM,cACJ,SACA,cACA,QACwB;AACxB,UAAM,QAAQ,KAAK,eAAe,YAAY;AAC9C,QAAI,CAAC,MAAO,QAAO;AAEnB,UAAM,UAAU,cAAc,MAAM,OAAO,MAAM,WAAW,MAAM,KAAK,SAAS,QAAQ,CAAC;AAEzF,QAAI;AACF,YAAM,cAAc,MAAM,QAAQ,SAAS,UAAU,mBAAmB;AAAA,QACtE,UAAU;AAAA,QACV,QAAQ,UAAU;AAAA,MACpB,CAAC;AACD,aAAO,OAAO,gBAAgB,WAAW,cAAc;AAAA,IACzD,SAAS,KAAK;AACZ,cAAQ,MAAM,6BAA6B,GAAG;AAC9C,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,cAA4B;AAChC,SAAK,OAAO,OAAO,YAAY;AAAA,EACjC;AACF;","names":[]}
@@ -0,0 +1,50 @@
1
+ import type { IAgentRuntime, UUID } from "@elizaos/core";
2
+ import { Service } from "@elizaos/core";
3
+ import { type WebSocket } from "ws";
4
+ import { type XRDeviceType, type XRPanelConfig, type XRServerControl } from "../protocol.ts";
5
+ import { VisionPipeline } from "./vision-pipeline.ts";
6
+ export declare const XR_SERVICE_TYPE = "xr-session";
7
+ export declare const XR_WS_PORT_DEFAULT = 31338;
8
+ export declare const XR_WS_PORT_ENV = "XR_WS_PORT";
9
+ export interface XRConnection {
10
+ id: string;
11
+ ws: WebSocket;
12
+ deviceType: XRDeviceType;
13
+ entityId: UUID;
14
+ roomId: UUID;
15
+ connectedAt: Date;
16
+ }
17
+ export declare class XRSessionService extends Service {
18
+ static serviceType: string;
19
+ readonly capabilityDescription = "Streams audio and camera from XR headsets (Quest 3, XReal) to the agent and returns voice responses.";
20
+ private wss;
21
+ private connections;
22
+ private audioPipeline;
23
+ private visionPipeline;
24
+ static start(runtime: IAgentRuntime): Promise<Service>;
25
+ private initialize;
26
+ stop(): Promise<void>;
27
+ getConnections(): XRConnection[];
28
+ hasActiveConnections(): boolean;
29
+ getVisionPipeline(): VisionPipeline;
30
+ sendText(connectionId: string, text: string): void;
31
+ sendControl(connectionId: string, msg: XRServerControl): void;
32
+ broadcastControl(msg: XRServerControl): void;
33
+ openView(connectionId: string, viewId: string, agentBaseUrl: string, config?: XRPanelConfig): void;
34
+ closeView(connectionId: string, viewId: string): void;
35
+ switchView(connectionId: string, viewId: string): void;
36
+ resizeView(connectionId: string, viewId: string, config: XRPanelConfig): void;
37
+ sendViewsCatalog(connectionId: string, views: Array<{
38
+ id: string;
39
+ label: string;
40
+ icon?: string;
41
+ description?: string;
42
+ }>): void;
43
+ sendAudio(connectionId: string, audio: Buffer, sampleRate?: number): void;
44
+ private onConnect;
45
+ private handleTextMessage;
46
+ private handleBinaryMessage;
47
+ private handleTranscript;
48
+ private ensureEntities;
49
+ }
50
+ //# sourceMappingURL=xr-session-service.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xr-session-service.d.ts","sourceRoot":"","sources":["../../src/services/xr-session-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,aAAa,EAEb,IAAI,EACL,MAAM,eAAe,CAAC;AACvB,OAAO,EAIL,OAAO,EACR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,SAAS,EAAmB,MAAM,IAAI,CAAC;AACrD,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,eAAO,MAAM,eAAe,eAAe,CAAC;AAC5C,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,cAAc,eAAe,CAAC;AAE3C,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,SAAS,CAAC;IACd,UAAU,EAAE,YAAY,CAAC;IACzB,QAAQ,EAAE,IAAI,CAAC;IACf,MAAM,EAAE,IAAI,CAAC;IACb,WAAW,EAAE,IAAI,CAAC;CACnB;AAED,qBAAa,gBAAiB,SAAQ,OAAO;IAC3C,OAAgB,WAAW,SAAmB;IAE9C,QAAQ,CAAC,qBAAqB,0GAC2E;IAEzG,OAAO,CAAC,GAAG,CAAmB;IAC9B,OAAO,CAAC,WAAW,CAAmC;IACtD,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,cAAc,CAAkB;WAElB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;YAMvD,UAAU;IAuBT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpC,cAAc,IAAI,YAAY,EAAE;IAIhC,oBAAoB,IAAI,OAAO;IAI/B,iBAAiB,IAAI,cAAc;IAInC,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAMlD,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,eAAe,GAAG,IAAI;IAM7D,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI;IAQ5C,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,aAAa,GACrB,IAAI;IASP,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAIrD,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAItD,UAAU,CACR,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,IAAI;IAIP,gBAAgB,CACd,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,GACD,IAAI;IAIP,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,SAAQ,GAAG,IAAI;IAcxE,OAAO,CAAC,SAAS;IAkCjB,OAAO,CAAC,iBAAiB;IAmDzB,OAAO,CAAC,mBAAmB;YAmBb,gBAAgB;YA8EhB,cAAc;CA0B7B"}