@elizaos/plugin-xr 2.0.3-beta.5 → 2.0.3-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/xr-query-vision.d.ts +3 -0
- package/dist/actions/xr-query-vision.d.ts.map +1 -0
- package/dist/actions/xr-query-vision.js +39 -0
- package/dist/actions/xr-query-vision.js.map +1 -0
- package/dist/actions/xr-view-actions.d.ts +18 -0
- package/dist/actions/xr-view-actions.d.ts.map +1 -0
- package/dist/actions/xr-view-actions.js +304 -0
- package/dist/actions/xr-view-actions.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +57 -0
- package/dist/index.js.map +1 -0
- package/dist/protocol.d.ts +124 -0
- package/dist/protocol.d.ts.map +1 -0
- package/dist/protocol.js +18 -0
- package/dist/protocol.js.map +1 -0
- package/dist/providers/xr-context.d.ts +3 -0
- package/dist/providers/xr-context.d.ts.map +1 -0
- package/dist/providers/xr-context.js +34 -0
- package/dist/providers/xr-context.js.map +1 -0
- package/dist/routes/xr-connect.d.ts +3 -0
- package/dist/routes/xr-connect.d.ts.map +1 -0
- package/{src/routes/xr-connect.ts → dist/routes/xr-connect.js} +12 -15
- package/dist/routes/xr-connect.js.map +1 -0
- package/dist/routes/xr-simulator-route.d.ts +8 -0
- package/dist/routes/xr-simulator-route.d.ts.map +1 -0
- package/{src/routes/xr-simulator-route.ts → dist/routes/xr-simulator-route.js} +10 -16
- package/dist/routes/xr-simulator-route.js.map +1 -0
- package/dist/routes/xr-status.d.ts +3 -0
- package/dist/routes/xr-status.d.ts.map +1 -0
- package/{src/routes/xr-status.ts → dist/routes/xr-status.js} +13 -15
- package/dist/routes/xr-status.js.map +1 -0
- package/dist/routes/xr-view-host.d.ts +24 -0
- package/dist/routes/xr-view-host.d.ts.map +1 -0
- package/{src/routes/xr-view-host.ts → dist/routes/xr-view-host.js} +22 -59
- package/dist/routes/xr-view-host.js.map +1 -0
- package/dist/routes/xr-views.d.ts +8 -0
- package/dist/routes/xr-views.d.ts.map +1 -0
- package/dist/routes/xr-views.js +31 -0
- package/dist/routes/xr-views.js.map +1 -0
- package/dist/services/audio-pipeline.d.ts +20 -0
- package/dist/services/audio-pipeline.d.ts.map +1 -0
- package/{src/services/audio-pipeline.ts → dist/services/audio-pipeline.js} +25 -58
- package/dist/services/audio-pipeline.js.map +1 -0
- package/dist/services/vision-pipeline.d.ts +16 -0
- package/dist/services/vision-pipeline.d.ts.map +1 -0
- package/dist/services/vision-pipeline.js +39 -0
- package/dist/services/vision-pipeline.js.map +1 -0
- package/dist/services/xr-session-service.d.ts +50 -0
- package/dist/services/xr-session-service.d.ts.map +1 -0
- package/{src/services/xr-session-service.ts → dist/services/xr-session-service.js} +85 -194
- package/dist/services/xr-session-service.js.map +1 -0
- package/package.json +9 -4
- package/AGENTS.md +0 -151
- package/CLAUDE.md +0 -151
- package/simulator/bun.lock +0 -159
- package/simulator/package.json +0 -28
- package/simulator/src/emulator.ts +0 -174
- package/simulator/src/mock-agent.ts +0 -233
- package/simulator/src/node.ts +0 -9
- package/simulator/src/playwright-fixture.ts +0 -169
- package/simulator/src/types.ts +0 -51
- package/simulator/tsconfig.json +0 -13
- package/simulator/vite.config.ts +0 -25
- package/src/__tests__/audio-pipeline.test.ts +0 -129
- package/src/__tests__/protocol.test.ts +0 -53
- package/src/__tests__/routes-e2e.test.ts +0 -276
- package/src/__tests__/vision-pipeline.test.ts +0 -73
- package/src/__tests__/xr-bundle-coverage.test.ts +0 -303
- package/src/__tests__/xr-feature-parity.test.ts +0 -524
- package/src/__tests__/xr-functional-parity.test.ts +0 -522
- package/src/__tests__/xr-view-host-http.test.ts +0 -239
- package/src/__tests__/xr-view-host.test.ts +0 -174
- package/src/actions/xr-query-vision.ts +0 -64
- package/src/actions/xr-view-actions.ts +0 -386
- package/src/index.ts +0 -55
- package/src/protocol.ts +0 -126
- package/src/providers/xr-context.ts +0 -49
- package/src/routes/xr-views.ts +0 -43
- package/src/services/vision-pipeline.ts +0 -57
- package/tsconfig.build.json +0 -9
- package/tsconfig.json +0 -30
- package/vitest.config.ts +0 -21
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import type { IAgentRuntime } from "@elizaos/core";
|
|
2
1
|
import { ModelType } from "@elizaos/core";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
// Prepend a RIFF/WAV header so Whisper can decode raw Float32 PCM.
|
|
6
|
-
function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
|
|
7
|
-
const channels = 1; // ScriptProcessorNode fallback is always mono
|
|
2
|
+
function pcmF32ToWav(pcmData, sampleRate) {
|
|
3
|
+
const channels = 1;
|
|
8
4
|
const dataSize = pcmData.length;
|
|
9
5
|
const header = Buffer.alloc(44);
|
|
10
6
|
header.write("RIFF", 0);
|
|
@@ -12,7 +8,7 @@ function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
|
|
|
12
8
|
header.write("WAVE", 8);
|
|
13
9
|
header.write("fmt ", 12);
|
|
14
10
|
header.writeUInt32LE(16, 16);
|
|
15
|
-
header.writeUInt16LE(3, 20);
|
|
11
|
+
header.writeUInt16LE(3, 20);
|
|
16
12
|
header.writeUInt16LE(channels, 22);
|
|
17
13
|
header.writeUInt32LE(sampleRate, 24);
|
|
18
14
|
header.writeUInt32LE(sampleRate * channels * 4, 28);
|
|
@@ -22,33 +18,17 @@ function pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {
|
|
|
22
18
|
header.writeUInt32LE(dataSize, 40);
|
|
23
19
|
return Buffer.concat([header, pcmData]);
|
|
24
20
|
}
|
|
25
|
-
|
|
26
|
-
// Accumulate up to FLUSH_AFTER_MS of audio then transcribe.
|
|
27
|
-
// Also flush if no chunk arrives within SILENCE_GAP_MS (end-of-utterance).
|
|
28
|
-
const FLUSH_AFTER_MS = 2000;
|
|
21
|
+
const FLUSH_AFTER_MS = 2e3;
|
|
29
22
|
const SILENCE_GAP_MS = 1500;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
export class AudioPipeline {
|
|
41
|
-
private pending = new Map<string, PendingTranscription>();
|
|
42
|
-
|
|
43
|
-
constructor(
|
|
44
|
-
private readonly runtime: IAgentRuntime,
|
|
45
|
-
private readonly onTranscript: (
|
|
46
|
-
connectionId: string,
|
|
47
|
-
text: string,
|
|
48
|
-
) => Promise<void>,
|
|
49
|
-
) {}
|
|
50
|
-
|
|
51
|
-
push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void {
|
|
23
|
+
class AudioPipeline {
|
|
24
|
+
constructor(runtime, onTranscript) {
|
|
25
|
+
this.runtime = runtime;
|
|
26
|
+
this.onTranscript = onTranscript;
|
|
27
|
+
}
|
|
28
|
+
runtime;
|
|
29
|
+
onTranscript;
|
|
30
|
+
pending = /* @__PURE__ */ new Map();
|
|
31
|
+
push(connectionId, header, chunk) {
|
|
52
32
|
let state = this.pending.get(connectionId);
|
|
53
33
|
if (!state) {
|
|
54
34
|
state = {
|
|
@@ -56,65 +36,52 @@ export class AudioPipeline {
|
|
|
56
36
|
firstTs: header.ts,
|
|
57
37
|
lastTs: header.ts,
|
|
58
38
|
encoding: header.encoding,
|
|
59
|
-
sampleRate: header.sampleRate
|
|
39
|
+
sampleRate: header.sampleRate
|
|
60
40
|
};
|
|
61
41
|
this.pending.set(connectionId, state);
|
|
62
42
|
}
|
|
63
|
-
|
|
64
43
|
state.chunks.push(chunk);
|
|
65
44
|
state.lastTs = header.ts;
|
|
66
|
-
|
|
67
|
-
// Reset silence timer on each incoming chunk
|
|
68
45
|
if (state.silenceTimer) clearTimeout(state.silenceTimer);
|
|
69
46
|
state.silenceTimer = setTimeout(
|
|
70
47
|
() => void this.flush(connectionId),
|
|
71
|
-
SILENCE_GAP_MS
|
|
48
|
+
SILENCE_GAP_MS
|
|
72
49
|
);
|
|
73
|
-
|
|
74
|
-
// Also flush if we've accumulated enough audio
|
|
75
50
|
if (state.lastTs - state.firstTs >= FLUSH_AFTER_MS) {
|
|
76
51
|
void this.flush(connectionId);
|
|
77
52
|
}
|
|
78
53
|
}
|
|
79
|
-
|
|
80
|
-
async flush(connectionId: string): Promise<void> {
|
|
54
|
+
async flush(connectionId) {
|
|
81
55
|
const state = this.pending.get(connectionId);
|
|
82
56
|
if (!state || state.chunks.length === 0) return;
|
|
83
|
-
|
|
84
57
|
if (state.silenceTimer) {
|
|
85
58
|
clearTimeout(state.silenceTimer);
|
|
86
|
-
state.silenceTimer =
|
|
59
|
+
state.silenceTimer = void 0;
|
|
87
60
|
}
|
|
88
61
|
this.pending.delete(connectionId);
|
|
89
|
-
|
|
90
62
|
const combined = Buffer.concat(state.chunks);
|
|
91
|
-
if (combined.length < 512) return;
|
|
92
|
-
|
|
93
|
-
// pcm-f32 is raw Float32 samples (mono, from ScriptProcessorNode fallback).
|
|
94
|
-
// Whisper expects a valid audio container — wrap with a WAV header.
|
|
95
|
-
const audioBuffer =
|
|
96
|
-
state.encoding === "pcm-f32"
|
|
97
|
-
? pcmF32ToWav(combined, state.sampleRate)
|
|
98
|
-
: combined;
|
|
99
|
-
|
|
63
|
+
if (combined.length < 512) return;
|
|
64
|
+
const audioBuffer = state.encoding === "pcm-f32" ? pcmF32ToWav(combined, state.sampleRate) : combined;
|
|
100
65
|
try {
|
|
101
66
|
const transcript = await this.runtime.useModel(
|
|
102
67
|
ModelType.TRANSCRIPTION,
|
|
103
|
-
audioBuffer
|
|
68
|
+
audioBuffer
|
|
104
69
|
);
|
|
105
70
|
const text = typeof transcript === "string" ? transcript.trim() : "";
|
|
106
71
|
if (text.length > 0) {
|
|
107
72
|
await this.onTranscript(connectionId, text);
|
|
108
73
|
}
|
|
109
74
|
} catch (err) {
|
|
110
|
-
// log but don't crash the pipeline
|
|
111
75
|
console.error("[plugin-xr] transcription error:", err);
|
|
112
76
|
}
|
|
113
77
|
}
|
|
114
|
-
|
|
115
|
-
clear(connectionId: string): void {
|
|
78
|
+
clear(connectionId) {
|
|
116
79
|
const state = this.pending.get(connectionId);
|
|
117
80
|
if (state?.silenceTimer) clearTimeout(state.silenceTimer);
|
|
118
81
|
this.pending.delete(connectionId);
|
|
119
82
|
}
|
|
120
83
|
}
|
|
84
|
+
export {
|
|
85
|
+
AudioPipeline
|
|
86
|
+
};
|
|
87
|
+
//# sourceMappingURL=audio-pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/services/audio-pipeline.ts"],"sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { ModelType } from \"@elizaos/core\";\nimport type { XRAudioHeader } from \"../protocol.js\";\n\n// Prepend a RIFF/WAV header so Whisper can decode raw Float32 PCM.\nfunction pcmF32ToWav(pcmData: Buffer, sampleRate: number): Buffer {\n const channels = 1; // ScriptProcessorNode fallback is always mono\n const dataSize = pcmData.length;\n const header = Buffer.alloc(44);\n header.write(\"RIFF\", 0);\n header.writeUInt32LE(36 + dataSize, 4);\n header.write(\"WAVE\", 8);\n header.write(\"fmt \", 12);\n header.writeUInt32LE(16, 16);\n header.writeUInt16LE(3, 20); // IEEE_FLOAT\n header.writeUInt16LE(channels, 22);\n header.writeUInt32LE(sampleRate, 24);\n header.writeUInt32LE(sampleRate * channels * 4, 28);\n header.writeUInt16LE(channels * 4, 32);\n header.writeUInt16LE(32, 34);\n header.write(\"data\", 36);\n header.writeUInt32LE(dataSize, 40);\n return Buffer.concat([header, pcmData]);\n}\n\n// Accumulate up to FLUSH_AFTER_MS of audio then transcribe.\n// Also flush if no chunk arrives within SILENCE_GAP_MS (end-of-utterance).\nconst FLUSH_AFTER_MS = 2000;\nconst SILENCE_GAP_MS = 1500;\n\nexport interface PendingTranscription {\n chunks: Buffer[];\n firstTs: number;\n lastTs: number;\n encoding: XRAudioHeader[\"encoding\"];\n sampleRate: number;\n silenceTimer?: ReturnType<typeof setTimeout>;\n}\n\nexport class AudioPipeline {\n private pending = new Map<string, PendingTranscription>();\n\n constructor(\n private readonly runtime: IAgentRuntime,\n private readonly onTranscript: (\n connectionId: string,\n text: string,\n ) => Promise<void>,\n ) {}\n\n push(connectionId: string, header: XRAudioHeader, chunk: Buffer): void {\n let state = this.pending.get(connectionId);\n if (!state) {\n state = {\n chunks: [],\n firstTs: header.ts,\n lastTs: header.ts,\n encoding: header.encoding,\n sampleRate: header.sampleRate,\n };\n this.pending.set(connectionId, state);\n }\n\n state.chunks.push(chunk);\n state.lastTs = header.ts;\n\n // Reset silence timer on each incoming chunk\n if (state.silenceTimer) clearTimeout(state.silenceTimer);\n state.silenceTimer = setTimeout(\n () => void this.flush(connectionId),\n SILENCE_GAP_MS,\n );\n\n // Also flush if we've accumulated enough audio\n if (state.lastTs - state.firstTs >= FLUSH_AFTER_MS) {\n void this.flush(connectionId);\n }\n }\n\n async flush(connectionId: string): Promise<void> {\n const state = this.pending.get(connectionId);\n if (!state || state.chunks.length === 0) return;\n\n if (state.silenceTimer) {\n clearTimeout(state.silenceTimer);\n state.silenceTimer = undefined;\n }\n this.pending.delete(connectionId);\n\n const combined = Buffer.concat(state.chunks);\n if (combined.length < 512) return; // too small to be real speech\n\n // pcm-f32 is raw Float32 samples (mono, from ScriptProcessorNode fallback).\n // Whisper expects a valid audio container — wrap with a WAV header.\n const audioBuffer =\n state.encoding === \"pcm-f32\"\n ? pcmF32ToWav(combined, state.sampleRate)\n : combined;\n\n try {\n const transcript = await this.runtime.useModel(\n ModelType.TRANSCRIPTION,\n audioBuffer,\n );\n const text = typeof transcript === \"string\" ? transcript.trim() : \"\";\n if (text.length > 0) {\n await this.onTranscript(connectionId, text);\n }\n } catch (err) {\n // log but don't crash the pipeline\n console.error(\"[plugin-xr] transcription error:\", err);\n }\n }\n\n clear(connectionId: string): void {\n const state = this.pending.get(connectionId);\n if (state?.silenceTimer) clearTimeout(state.silenceTimer);\n this.pending.delete(connectionId);\n }\n}\n"],"mappings":"AACA,SAAS,iBAAiB;AAI1B,SAAS,YAAY,SAAiB,YAA4B;AAChE,QAAM,WAAW;AACjB,QAAM,WAAW,QAAQ;AACzB,QAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,SAAO,MAAM,QAAQ,CAAC;AACtB,SAAO,cAAc,KAAK,UAAU,CAAC;AACrC,SAAO,MAAM,QAAQ,CAAC;AACtB,SAAO,MAAM,QAAQ,EAAE;AACvB,SAAO,cAAc,IAAI,EAAE;AAC3B,SAAO,cAAc,GAAG,EAAE;AAC1B,SAAO,cAAc,UAAU,EAAE;AACjC,SAAO,cAAc,YAAY,EAAE;AACnC,SAAO,cAAc,aAAa,WAAW,GAAG,EAAE;AAClD,SAAO,cAAc,WAAW,GAAG,EAAE;AACrC,SAAO,cAAc,IAAI,EAAE;AAC3B,SAAO,MAAM,QAAQ,EAAE;AACvB,SAAO,cAAc,UAAU,EAAE;AACjC,SAAO,OAAO,OAAO,CAAC,QAAQ,OAAO,CAAC;AACxC;AAIA,MAAM,iBAAiB;AACvB,MAAM,iBAAiB;AAWhB,MAAM,cAAc;AAAA,EAGzB,YACmB,SACA,cAIjB;AALiB;AACA;AAAA,EAIhB;AAAA,EALgB;AAAA,EACA;AAAA,EAJX,UAAU,oBAAI,IAAkC;AAAA,EAUxD,KAAK,cAAsB,QAAuB,OAAqB;AACrE,QAAI,QAAQ,KAAK,QAAQ,IAAI,YAAY;AACzC,QAAI,CAAC,OAAO;AACV,cAAQ;AAAA,QACN,QAAQ,CAAC;AAAA,QACT,SAAS,OAAO;AAAA,QAChB,QAAQ,OAAO;AAAA,QACf,UAAU,OAAO;AAAA,QACjB,YAAY,OAAO;AAAA,MACrB;AACA,WAAK,QAAQ,IAAI,cAAc,KAAK;AAAA,IACtC;AAEA,UAAM,OAAO,KAAK,KAAK;AACvB,UAAM,SAAS,OAAO;AAGtB,QAAI,MAAM,aAAc,cAAa,MAAM,YAAY;AACvD,UAAM,eAAe;AAAA,MACnB,MAAM,KAAK,KAAK,MAAM,YAAY;AAAA,MAClC;AAAA,IACF;AAGA,QAAI,MAAM,SAAS,MAAM,WAAW,gBAAgB;AAClD,WAAK,KAAK,MAAM,YAAY;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,MAAM,MAAM,cAAqC;AAC/C,UAAM,QAAQ,KAAK,QAAQ,IAAI,YAAY;AAC3C,QAAI,CAAC,SAAS,MAAM,OAAO,WAAW,EAAG;AAEzC,QAAI,MAAM,cAAc;AACtB,mBAAa,MAAM,YAAY;AAC/B,YAAM,eAAe;AAAA,IACvB;AACA,SAAK,QAAQ,OAAO,YAAY;AAEhC,UAAM,WAAW,OAAO,OAAO,MAAM,MAAM;AAC3C,QAAI,SAAS,SAAS,IAAK;AAI3B,UAAM,cACJ,MAAM,aAAa,YACf,YAAY,UAAU,MAAM,UAAU,IACtC;AAEN,QAAI;AACF,YAAM,aAAa,MAAM,KAAK,QAAQ;AAAA,QACpC,UAAU;AAAA,QACV;AAAA,MACF;AACA,YAAM,OAAO,OAAO,eAAe,WAAW,WAAW,KAAK,IAAI;AAClE,UAAI,KAAK,SAAS,GAAG;AACnB,cAAM,KAAK,aAAa,cAAc,IAAI;AAAA,MAC5C;AAAA,IACF,SAAS,KAAK;AAEZ,cAAQ,MAAM,oCAAoC,GAAG;AAAA,IACvD;AAAA,EACF;AAAA,EAEA,MAAM,cAA4B;AAChC,UAAM,QAAQ,KAAK,QAAQ,IAAI,YAAY;AAC3C,QAAI,OAAO,aAAc,cAAa,MAAM,YAAY;AACxD,SAAK,QAAQ,OAAO,YAAY;AAAA,EAClC;AACF;","names":[]}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { IAgentRuntime } from "@elizaos/core";
|
|
2
|
+
import type { XRFrameHeader } from "../protocol.ts";
|
|
3
|
+
export interface LatestFrame {
|
|
4
|
+
data: Buffer;
|
|
5
|
+
header: XRFrameHeader;
|
|
6
|
+
receivedAt: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class VisionPipeline {
|
|
9
|
+
private latest;
|
|
10
|
+
storeFrame(connectionId: string, header: XRFrameHeader, data: Buffer): void;
|
|
11
|
+
getLatestFrame(connectionId: string): LatestFrame | undefined;
|
|
12
|
+
hasRecentFrame(connectionId: string): boolean;
|
|
13
|
+
describeFrame(runtime: IAgentRuntime, connectionId: string, prompt?: string): Promise<string | null>;
|
|
14
|
+
clear(connectionId: string): void;
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=vision-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vision-pipeline.d.ts","sourceRoot":"","sources":["../../src/services/vision-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAkC;IAEhD,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAI3E,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAO7D,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAIvC,aAAa,CACjB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAkBzB,KAAK,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;CAGlC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { ModelType } from "@elizaos/core";
|
|
2
|
+
const FRAME_MAX_AGE_MS = 1e4;
|
|
3
|
+
class VisionPipeline {
|
|
4
|
+
latest = /* @__PURE__ */ new Map();
|
|
5
|
+
storeFrame(connectionId, header, data) {
|
|
6
|
+
this.latest.set(connectionId, { data, header, receivedAt: Date.now() });
|
|
7
|
+
}
|
|
8
|
+
getLatestFrame(connectionId) {
|
|
9
|
+
const frame = this.latest.get(connectionId);
|
|
10
|
+
if (!frame) return void 0;
|
|
11
|
+
if (Date.now() - frame.receivedAt > FRAME_MAX_AGE_MS) return void 0;
|
|
12
|
+
return frame;
|
|
13
|
+
}
|
|
14
|
+
hasRecentFrame(connectionId) {
|
|
15
|
+
return this.getLatestFrame(connectionId) !== void 0;
|
|
16
|
+
}
|
|
17
|
+
async describeFrame(runtime, connectionId, prompt) {
|
|
18
|
+
const frame = this.getLatestFrame(connectionId);
|
|
19
|
+
if (!frame) return null;
|
|
20
|
+
const dataUrl = `data:image/${frame.header.format};base64,${frame.data.toString("base64")}`;
|
|
21
|
+
try {
|
|
22
|
+
const description = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, {
|
|
23
|
+
imageUrl: dataUrl,
|
|
24
|
+
prompt: prompt ?? "Describe what you see in this image concisely."
|
|
25
|
+
});
|
|
26
|
+
return typeof description === "string" ? description : null;
|
|
27
|
+
} catch (err) {
|
|
28
|
+
console.error("[plugin-xr] vision error:", err);
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
clear(connectionId) {
|
|
33
|
+
this.latest.delete(connectionId);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export {
|
|
37
|
+
VisionPipeline
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=vision-pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/services/vision-pipeline.ts"],"sourcesContent":["import type { IAgentRuntime } from \"@elizaos/core\";\nimport { ModelType } from \"@elizaos/core\";\nimport type { XRFrameHeader } from \"../protocol.js\";\n\nexport interface LatestFrame {\n data: Buffer;\n header: XRFrameHeader;\n receivedAt: number;\n}\n\n// A frame older than this is considered stale and won't be described\nconst FRAME_MAX_AGE_MS = 10_000;\n\nexport class VisionPipeline {\n private latest = new Map<string, LatestFrame>();\n\n storeFrame(connectionId: string, header: XRFrameHeader, data: Buffer): void {\n this.latest.set(connectionId, { data, header, receivedAt: Date.now() });\n }\n\n getLatestFrame(connectionId: string): LatestFrame | undefined {\n const frame = this.latest.get(connectionId);\n if (!frame) return undefined;\n if (Date.now() - frame.receivedAt > FRAME_MAX_AGE_MS) return undefined;\n return frame;\n }\n\n hasRecentFrame(connectionId: string): boolean {\n return this.getLatestFrame(connectionId) !== undefined;\n }\n\n async describeFrame(\n runtime: IAgentRuntime,\n connectionId: string,\n prompt?: string,\n ): Promise<string | null> {\n const frame = this.getLatestFrame(connectionId);\n if (!frame) return null;\n\n const dataUrl = `data:image/${frame.header.format};base64,${frame.data.toString(\"base64\")}`;\n\n try {\n const description = await runtime.useModel(ModelType.IMAGE_DESCRIPTION, {\n imageUrl: dataUrl,\n prompt: prompt ?? \"Describe what you see in this image concisely.\",\n });\n return typeof description === \"string\" ? description : null;\n } catch (err) {\n console.error(\"[plugin-xr] vision error:\", err);\n return null;\n }\n }\n\n clear(connectionId: string): void {\n this.latest.delete(connectionId);\n }\n}\n"],"mappings":"AACA,SAAS,iBAAiB;AAU1B,MAAM,mBAAmB;AAElB,MAAM,eAAe;AAAA,EAClB,SAAS,oBAAI,IAAyB;AAAA,EAE9C,WAAW,cAAsB,QAAuB,MAAoB;AAC1E,SAAK,OAAO,IAAI,cAAc,EAAE,MAAM,QAAQ,YAAY,KAAK,IAAI,EAAE,CAAC;AAAA,EACxE;AAAA,EAEA,eAAe,cAA+C;AAC5D,UAAM,QAAQ,KAAK,OAAO,IAAI,YAAY;AAC1C,QAAI,CAAC,MAAO,QAAO;AACnB,QAAI,KAAK,IAAI,IAAI,MAAM,aAAa,iBAAkB,QAAO;AAC7D,WAAO;AAAA,EACT;AAAA,EAEA,eAAe,cAA+B;AAC5C,WAAO,KAAK,eAAe,YAAY,MAAM;AAAA,EAC/C;AAAA,EAEA,MAAM,cACJ,SACA,cACA,QACwB;AACxB,UAAM,QAAQ,KAAK,eAAe,YAAY;AAC9C,QAAI,CAAC,MAAO,QAAO;AAEnB,UAAM,UAAU,cAAc,MAAM,OAAO,MAAM,WAAW,MAAM,KAAK,SAAS,QAAQ,CAAC;AAEzF,QAAI;AACF,YAAM,cAAc,MAAM,QAAQ,SAAS,UAAU,mBAAmB;AAAA,QACtE,UAAU;AAAA,QACV,QAAQ,UAAU;AAAA,MACpB,CAAC;AACD,aAAO,OAAO,gBAAgB,WAAW,cAAc;AAAA,IACzD,SAAS,KAAK;AACZ,cAAQ,MAAM,6BAA6B,GAAG;AAC9C,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,cAA4B;AAChC,SAAK,OAAO,OAAO,YAAY;AAAA,EACjC;AACF;","names":[]}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { IAgentRuntime, UUID } from "@elizaos/core";
|
|
2
|
+
import { Service } from "@elizaos/core";
|
|
3
|
+
import { type WebSocket } from "ws";
|
|
4
|
+
import { type XRDeviceType, type XRPanelConfig, type XRServerControl } from "../protocol.ts";
|
|
5
|
+
import { VisionPipeline } from "./vision-pipeline.ts";
|
|
6
|
+
export declare const XR_SERVICE_TYPE = "xr-session";
|
|
7
|
+
export declare const XR_WS_PORT_DEFAULT = 31338;
|
|
8
|
+
export declare const XR_WS_PORT_ENV = "XR_WS_PORT";
|
|
9
|
+
export interface XRConnection {
|
|
10
|
+
id: string;
|
|
11
|
+
ws: WebSocket;
|
|
12
|
+
deviceType: XRDeviceType;
|
|
13
|
+
entityId: UUID;
|
|
14
|
+
roomId: UUID;
|
|
15
|
+
connectedAt: Date;
|
|
16
|
+
}
|
|
17
|
+
export declare class XRSessionService extends Service {
|
|
18
|
+
static serviceType: string;
|
|
19
|
+
readonly capabilityDescription = "Streams audio and camera from XR headsets (Quest 3, XReal) to the agent and returns voice responses.";
|
|
20
|
+
private wss;
|
|
21
|
+
private connections;
|
|
22
|
+
private audioPipeline;
|
|
23
|
+
private visionPipeline;
|
|
24
|
+
static start(runtime: IAgentRuntime): Promise<Service>;
|
|
25
|
+
private initialize;
|
|
26
|
+
stop(): Promise<void>;
|
|
27
|
+
getConnections(): XRConnection[];
|
|
28
|
+
hasActiveConnections(): boolean;
|
|
29
|
+
getVisionPipeline(): VisionPipeline;
|
|
30
|
+
sendText(connectionId: string, text: string): void;
|
|
31
|
+
sendControl(connectionId: string, msg: XRServerControl): void;
|
|
32
|
+
broadcastControl(msg: XRServerControl): void;
|
|
33
|
+
openView(connectionId: string, viewId: string, agentBaseUrl: string, config?: XRPanelConfig): void;
|
|
34
|
+
closeView(connectionId: string, viewId: string): void;
|
|
35
|
+
switchView(connectionId: string, viewId: string): void;
|
|
36
|
+
resizeView(connectionId: string, viewId: string, config: XRPanelConfig): void;
|
|
37
|
+
sendViewsCatalog(connectionId: string, views: Array<{
|
|
38
|
+
id: string;
|
|
39
|
+
label: string;
|
|
40
|
+
icon?: string;
|
|
41
|
+
description?: string;
|
|
42
|
+
}>): void;
|
|
43
|
+
sendAudio(connectionId: string, audio: Buffer, sampleRate?: number): void;
|
|
44
|
+
private onConnect;
|
|
45
|
+
private handleTextMessage;
|
|
46
|
+
private handleBinaryMessage;
|
|
47
|
+
private handleTranscript;
|
|
48
|
+
private ensureEntities;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=xr-session-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xr-session-service.d.ts","sourceRoot":"","sources":["../../src/services/xr-session-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,aAAa,EAEb,IAAI,EACL,MAAM,eAAe,CAAC;AACvB,OAAO,EAIL,OAAO,EACR,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,SAAS,EAAmB,MAAM,IAAI,CAAC;AACrD,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,eAAO,MAAM,eAAe,eAAe,CAAC;AAC5C,eAAO,MAAM,kBAAkB,QAAQ,CAAC;AACxC,eAAO,MAAM,cAAc,eAAe,CAAC;AAE3C,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,SAAS,CAAC;IACd,UAAU,EAAE,YAAY,CAAC;IACzB,QAAQ,EAAE,IAAI,CAAC;IACf,MAAM,EAAE,IAAI,CAAC;IACb,WAAW,EAAE,IAAI,CAAC;CACnB;AAED,qBAAa,gBAAiB,SAAQ,OAAO;IAC3C,OAAgB,WAAW,SAAmB;IAE9C,QAAQ,CAAC,qBAAqB,0GAC2E;IAEzG,OAAO,CAAC,GAAG,CAAmB;IAC9B,OAAO,CAAC,WAAW,CAAmC;IACtD,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,cAAc,CAAkB;WAElB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;YAMvD,UAAU;IAuBT,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAYpC,cAAc,IAAI,YAAY,EAAE;IAIhC,oBAAoB,IAAI,OAAO;IAI/B,iBAAiB,IAAI,cAAc;IAInC,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAMlD,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,eAAe,GAAG,IAAI;IAM7D,gBAAgB,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI;IAQ5C,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,aAAa,GACrB,IAAI;IASP,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAIrD,UAAU,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAItD,UAAU,CACR,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,aAAa,GACpB,IAAI;IAIP,gBAAgB,CACd,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,GACD,IAAI;IAIP,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,SAAQ,GAAG,IAAI;IAcxE,OAAO,CAAC,SAAS;IAkCjB,OAAO,CAAC,iBAAiB;IAmDzB,OAAO,CAAC,mBAAmB;YAmBb,gBAAgB;YA8EhB,cAAc;CA0B7B"}
|