@voicethere/agent 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.js +24 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/protocol.d.ts +24 -2
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js.map +1 -1
- package/dist/runtime.d.ts +3 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +68 -1
- package/dist/runtime.js.map +1 -1
- package/dist/templates/echo/agent.js +26 -2
- package/dist/templates/echo-dc/agent.js +26 -2
- package/dist/templates/game-sync/agent.js +26 -2
- package/dist/templates/positional-tts/agent.js +26 -2
- package/dist/templates/recording-consent/agent.js +26 -2
- package/dist/templates/registry.d.ts.map +1 -1
- package/dist/templates/registry.js +2 -1
- package/dist/templates/registry.js.map +1 -1
- package/dist/templates/spatial-showcase/agent.js +255 -24
- package/dist/templates/voice-showcase/agent.js +26 -2
- package/dist/templates/voice-starter/agent.js +26 -2
- package/dist/templates/webhooks/agent.js +26 -2
- package/dist/templates/webhooks-redis/agent.js +26 -2
- package/package.json +4 -4
- package/templates/README.md +10 -10
- package/templates/spatial-showcase/agent.ts +188 -33
- package/templates/spatial-showcase/protocol.ts +3 -1
- package/templates/spatial-showcase/sine.ts +43 -0
|
@@ -10903,6 +10903,12 @@ function isPlayStopAckMessage(value) {
|
|
|
10903
10903
|
const msg = value;
|
|
10904
10904
|
return msg.type === "play_stop_ack" && typeof msg.requestId === "string";
|
|
10905
10905
|
}
|
|
10906
|
+
function isPlayPoseAckMessage(value) {
|
|
10907
|
+
if (!value || typeof value !== "object")
|
|
10908
|
+
return false;
|
|
10909
|
+
const msg = value;
|
|
10910
|
+
return msg.type === "play_pose_ack" && typeof msg.requestId === "string";
|
|
10911
|
+
}
|
|
10906
10912
|
function isRecordingControlAckMessage(value) {
|
|
10907
10913
|
if (!value || typeof value !== "object")
|
|
10908
10914
|
return false;
|
|
@@ -10963,7 +10969,7 @@ function isParentMessage(value) {
|
|
|
10963
10969
|
if (!value || typeof value !== "object")
|
|
10964
10970
|
return false;
|
|
10965
10971
|
const msg = value;
|
|
10966
|
-
return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
|
|
10972
|
+
return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "play_ack" || msg.type === "play_status_ack" || msg.type === "play_stop_ack" || msg.type === "play_pose_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
|
|
10967
10973
|
}
|
|
10968
10974
|
function isSessionScopedParentMessage(value) {
|
|
10969
10975
|
return isParentMessage(value) && !isWebhookMessage(value);
|
|
@@ -10985,6 +10991,7 @@ var pendingRecordingAcks = /* @__PURE__ */ new Map();
|
|
|
10985
10991
|
var pendingPlayAcks = /* @__PURE__ */ new Map();
|
|
10986
10992
|
var pendingPlayStatusAcks = /* @__PURE__ */ new Map();
|
|
10987
10993
|
var pendingPlayStopAcks = /* @__PURE__ */ new Map();
|
|
10994
|
+
var pendingPlayPoseAcks = /* @__PURE__ */ new Map();
|
|
10988
10995
|
var pendingMixAcks = /* @__PURE__ */ new Map();
|
|
10989
10996
|
var pendingSttAcks = /* @__PURE__ */ new Map();
|
|
10990
10997
|
function handleRecordingControlAck(message) {
|
|
@@ -11039,6 +11046,19 @@ function handlePlayStopAck(message) {
|
|
|
11039
11046
|
requestId: message.requestId
|
|
11040
11047
|
});
|
|
11041
11048
|
}
|
|
11049
|
+
function handlePlayPoseAck(message) {
|
|
11050
|
+
const pending = pendingPlayPoseAcks.get(message.requestId);
|
|
11051
|
+
if (!pending)
|
|
11052
|
+
return;
|
|
11053
|
+
clearTimeout(pending.timer);
|
|
11054
|
+
pendingPlayPoseAcks.delete(message.requestId);
|
|
11055
|
+
pending.resolve({
|
|
11056
|
+
ok: message.ok,
|
|
11057
|
+
playId: message.playId,
|
|
11058
|
+
reason: message.reason,
|
|
11059
|
+
requestId: message.requestId
|
|
11060
|
+
});
|
|
11061
|
+
}
|
|
11042
11062
|
function handleMixControlAck(message) {
|
|
11043
11063
|
const pending = pendingMixAcks.get(message.requestId);
|
|
11044
11064
|
if (!pending)
|
|
@@ -11124,7 +11144,7 @@ function sendParentMessage(message) {
|
|
|
11124
11144
|
process.send?.(message);
|
|
11125
11145
|
return;
|
|
11126
11146
|
}
|
|
11127
|
-
if (msgType === "play" || msgType === "play_status" || msgType === "play_stop") {
|
|
11147
|
+
if (msgType === "play" || msgType === "play_status" || msgType === "play_stop" || msgType === "play_pose") {
|
|
11128
11148
|
process.send?.(message);
|
|
11129
11149
|
return;
|
|
11130
11150
|
}
|
|
@@ -11310,6 +11330,10 @@ function defineAgent(handlers) {
|
|
|
11310
11330
|
handlePlayStopAck(message);
|
|
11311
11331
|
return;
|
|
11312
11332
|
}
|
|
11333
|
+
if (isPlayPoseAckMessage(message)) {
|
|
11334
|
+
handlePlayPoseAck(message);
|
|
11335
|
+
return;
|
|
11336
|
+
}
|
|
11313
11337
|
if (isMixControlAckMessage(message)) {
|
|
11314
11338
|
handleMixControlAck(message);
|
|
11315
11339
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voicethere/agent",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.12",
|
|
4
4
|
"description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -95,9 +95,9 @@
|
|
|
95
95
|
"@node-webrtc-rust/sdk": ">=0.9.0"
|
|
96
96
|
},
|
|
97
97
|
"devDependencies": {
|
|
98
|
-
"@node-webrtc-rust/helpers": "0.9.
|
|
99
|
-
"@node-webrtc-rust/sdk": "0.9.
|
|
100
|
-
"@node-webrtc-rust/signaling": "0.9.
|
|
98
|
+
"@node-webrtc-rust/helpers": "0.9.9",
|
|
99
|
+
"@node-webrtc-rust/sdk": "0.9.9",
|
|
100
|
+
"@node-webrtc-rust/signaling": "0.9.9",
|
|
101
101
|
"@types/node": "22.20.1",
|
|
102
102
|
"http-server": "14.1.1",
|
|
103
103
|
"ioredis": "5.11.1",
|
package/templates/README.md
CHANGED
|
@@ -15,10 +15,10 @@ import {
|
|
|
15
15
|
|
|
16
16
|
## Product vs e2e
|
|
17
17
|
|
|
18
|
-
| Kind | Dashboard create
|
|
19
|
-
| ----------- |
|
|
18
|
+
| Kind | Dashboard create | Prebuilt seed bundle | Typical consumer |
|
|
19
|
+
| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
|
|
20
20
|
| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `spatial-showcase`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
|
|
21
|
-
| **e2e** | No
|
|
21
|
+
| **e2e** | No | No — build from sources at test time | `voicethere/e2e` smokes |
|
|
22
22
|
|
|
23
23
|
Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
|
|
24
24
|
|
|
@@ -83,17 +83,17 @@ Sources: `positional-tts/agent.ts` (defineAgent wiring), `positional-tts/orbit.t
|
|
|
83
83
|
|
|
84
84
|
Spatial audio showcase for `/showcase` — one agent template with three demos selected by `{ type: "join", demo }` on the control DataChannel:
|
|
85
85
|
|
|
86
|
-
| Demo
|
|
87
|
-
|
|
|
88
|
-
| `orbit`
|
|
89
|
-
| `soundboard` | Positional clip pads — browser sends `clipId` only (never URLs); agent resolves against an allowlisted `assetOrigin`
|
|
90
|
-
| `proximity`
|
|
86
|
+
| Demo | Behavior |
|
|
87
|
+
| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
88
|
+
| `orbit` | Looping sine tone + TTS orbit the listener (`setPlayPose` + `setTtsPose`); STT echo off; `{ type: "orbit", action: "set" \| "say" }`; agent emits `{ type: "orbit_pose" }` |
|
|
89
|
+
| `soundboard` | Positional clip pads — browser sends `clipId` only (never URLs); agent resolves against an allowlisted `assetOrigin` |
|
|
90
|
+
| `proximity` | Shared mix room (`createMixGroup` + `setClientPose`); `{ type: "move" }`, `{ type: "mute_peer" }`; `{ type: "room_state" }` broadcast |
|
|
91
91
|
|
|
92
92
|
**Protocol:** inbound `join`, `orbit`, `pad`, `pad_stop`, `pad_status`, `move`, `mute_peer`, `leave`, `ping` — outbound `showcase_ack`, `orbit_pose`, `pad_progress`, `room_state`, `room_full`, `error`.
|
|
93
93
|
|
|
94
|
-
**Limits:** `MAX_ROOM_PEERS = 8`, `MAX_ACTIVE_PLAYS =
|
|
94
|
+
**Limits:** `MAX_ROOM_PEERS = 8`, `MAX_ACTIVE_PLAYS = 12` per session. Clip ids: `chime`, `bell`, `laser`, `impact`, `footsteps`, `rain-loop`, `cafe-loop`, `jingle`.
|
|
95
95
|
|
|
96
|
-
Sources: `spatial-showcase/agent.ts`, `protocol.ts`, `sounds.ts`, `room.ts`, `orbit.ts`.
|
|
96
|
+
Sources: `spatial-showcase/agent.ts`, `protocol.ts`, `sounds.ts`, `room.ts`, `orbit.ts`, `sine.ts`.
|
|
97
97
|
|
|
98
98
|
### `webhooks.ts` (`webhooks`)
|
|
99
99
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Spatial audio showcase —
|
|
2
|
+
* Spatial audio showcase — orbiting sine + TTS, positional soundboard, proximity room.
|
|
3
3
|
*
|
|
4
4
|
* Browser joins with `{ type: "join", demo, assetOrigin? }` then sends demo-specific
|
|
5
5
|
* commands. Clip URLs are resolved server-side from clipId + allowlisted assetOrigin.
|
|
@@ -21,7 +21,9 @@ import {
|
|
|
21
21
|
sendToClient,
|
|
22
22
|
setClientPose,
|
|
23
23
|
setListenerMute,
|
|
24
|
+
setPlayPose,
|
|
24
25
|
setPositionalMixing,
|
|
26
|
+
setSttEnabled,
|
|
25
27
|
setTtsPose,
|
|
26
28
|
speak,
|
|
27
29
|
stopPlay,
|
|
@@ -42,16 +44,23 @@ import {
|
|
|
42
44
|
} from "./protocol.js";
|
|
43
45
|
import { ProximityRoom } from "./room.js";
|
|
44
46
|
import {
|
|
47
|
+
isShowcaseClipId,
|
|
45
48
|
resolveClipUrl,
|
|
46
49
|
SHOWCASE_SOUND_FILES,
|
|
47
50
|
type ShowcaseClipId,
|
|
48
51
|
} from "./sounds.js";
|
|
52
|
+
import {
|
|
53
|
+
buildOrbitSineInlineClipBase64,
|
|
54
|
+
ORBIT_SINE_INLINE_URL,
|
|
55
|
+
} from "./sine.js";
|
|
49
56
|
|
|
50
57
|
const ORBIT_TICK_MS = 50;
|
|
51
58
|
const ORBIT_POSE_EMIT_MS = 100;
|
|
52
59
|
const ROOM_STATE_TICK_MS = 100;
|
|
53
60
|
const LOOP_PAD_POLL_MS = 250;
|
|
54
|
-
const
|
|
61
|
+
const ORBIT_SINE_POLL_MS = 250;
|
|
62
|
+
const ORBIT_SINE_VOLUME = 0.25;
|
|
63
|
+
export const MAX_ACTIVE_PLAYS = 12;
|
|
55
64
|
const SHOWCASE_ROOM_GROUP_ID = "showcase-room";
|
|
56
65
|
|
|
57
66
|
type LoopPadState = {
|
|
@@ -73,6 +82,8 @@ type SessionState = {
|
|
|
73
82
|
playIds: Set<string>;
|
|
74
83
|
loopPads: Map<string, LoopPadState>;
|
|
75
84
|
loopPollTimers: Map<string, ReturnType<typeof setInterval>>;
|
|
85
|
+
orbitSinePlayId?: string;
|
|
86
|
+
orbitSinePollTimer?: ReturnType<typeof setInterval>;
|
|
76
87
|
inProximityRoom: boolean;
|
|
77
88
|
lastOrbitPoseEmitMs?: number;
|
|
78
89
|
};
|
|
@@ -95,11 +106,15 @@ function ensureShowcaseRoomGroup(): Promise<boolean> {
|
|
|
95
106
|
return showcaseRoomGroupReady;
|
|
96
107
|
}
|
|
97
108
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
109
|
+
type PadAckExtra = {
|
|
110
|
+
playId?: string;
|
|
111
|
+
clipId?: ShowcaseClipId;
|
|
112
|
+
x?: number;
|
|
113
|
+
z?: number;
|
|
114
|
+
placement?: PlayOptions["placement"];
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
function ack(sessionId: string, action: string, extra?: PadAckExtra): void {
|
|
103
118
|
sendToClient(sessionId, {
|
|
104
119
|
type: "showcase_ack",
|
|
105
120
|
action,
|
|
@@ -108,12 +123,18 @@ function ack(
|
|
|
108
123
|
});
|
|
109
124
|
}
|
|
110
125
|
|
|
111
|
-
function ackError(
|
|
126
|
+
function ackError(
|
|
127
|
+
sessionId: string,
|
|
128
|
+
action: string,
|
|
129
|
+
error: string,
|
|
130
|
+
extra?: { clipId?: ShowcaseClipId },
|
|
131
|
+
): void {
|
|
112
132
|
sendToClient(sessionId, {
|
|
113
133
|
type: "showcase_ack",
|
|
114
134
|
action,
|
|
115
135
|
ok: false,
|
|
116
136
|
error,
|
|
137
|
+
...(extra?.clipId !== undefined ? { clipId: extra.clipId } : {}),
|
|
117
138
|
});
|
|
118
139
|
}
|
|
119
140
|
|
|
@@ -143,7 +164,10 @@ async function safeMixCall(
|
|
|
143
164
|
}
|
|
144
165
|
}
|
|
145
166
|
|
|
146
|
-
function defaultSessionState(
|
|
167
|
+
function defaultSessionState(
|
|
168
|
+
demo: ShowcaseDemo,
|
|
169
|
+
assetOrigin?: string,
|
|
170
|
+
): SessionState {
|
|
147
171
|
return {
|
|
148
172
|
demo,
|
|
149
173
|
assetOrigin,
|
|
@@ -162,7 +186,24 @@ function clearRoomStateTimer(state: SessionState): void {
|
|
|
162
186
|
}
|
|
163
187
|
}
|
|
164
188
|
|
|
165
|
-
function
|
|
189
|
+
function stopLoopPadsForClip(
|
|
190
|
+
sessionId: string,
|
|
191
|
+
state: SessionState,
|
|
192
|
+
clipId: ShowcaseClipId,
|
|
193
|
+
): void {
|
|
194
|
+
for (const [playId, pad] of [...state.loopPads.entries()]) {
|
|
195
|
+
if (pad.clipId === clipId) {
|
|
196
|
+
void stopPlay(playId);
|
|
197
|
+
clearLoopPad(sessionId, state, playId);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function clearLoopPad(
|
|
203
|
+
sessionId: string,
|
|
204
|
+
state: SessionState,
|
|
205
|
+
playId: string,
|
|
206
|
+
): void {
|
|
166
207
|
const timer = state.loopPollTimers.get(playId);
|
|
167
208
|
if (timer) {
|
|
168
209
|
clearInterval(timer);
|
|
@@ -187,6 +228,87 @@ function clearAllPlays(sessionId: string, state: SessionState): void {
|
|
|
187
228
|
state.loopPollTimers.clear();
|
|
188
229
|
}
|
|
189
230
|
|
|
231
|
+
function currentOrbitPose(state: SessionState) {
|
|
232
|
+
if (!state.orbitStartMs) {
|
|
233
|
+
return orbitTtsPose(0, state.orbit);
|
|
234
|
+
}
|
|
235
|
+
const elapsedSec = (Date.now() - state.orbitStartMs) / 1000;
|
|
236
|
+
return orbitTtsPose(elapsedSec, state.orbit);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function clearOrbitSinePoll(state: SessionState): void {
|
|
240
|
+
if (state.orbitSinePollTimer) {
|
|
241
|
+
clearInterval(state.orbitSinePollTimer);
|
|
242
|
+
state.orbitSinePollTimer = undefined;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async function stopOrbitSine(state: SessionState): Promise<void> {
|
|
247
|
+
clearOrbitSinePoll(state);
|
|
248
|
+
const playId = state.orbitSinePlayId;
|
|
249
|
+
state.orbitSinePlayId = undefined;
|
|
250
|
+
if (playId) {
|
|
251
|
+
await stopPlay(playId);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function startOrbitSineLoopPoll(sessionId: string, state: SessionState): void {
|
|
256
|
+
clearOrbitSinePoll(state);
|
|
257
|
+
const playId = state.orbitSinePlayId;
|
|
258
|
+
if (!playId) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
state.orbitSinePollTimer = setInterval(() => {
|
|
263
|
+
void (async () => {
|
|
264
|
+
if (!state.orbitSinePlayId) {
|
|
265
|
+
clearOrbitSinePoll(state);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
const status = await getPlay(state.orbitSinePlayId);
|
|
269
|
+
if (!status.ok) {
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
if (status.status === "ended") {
|
|
273
|
+
const replay = await play({
|
|
274
|
+
url: ORBIT_SINE_INLINE_URL,
|
|
275
|
+
bytes: buildOrbitSineInlineClipBase64(),
|
|
276
|
+
sessionIds: [sessionId],
|
|
277
|
+
volume: ORBIT_SINE_VOLUME,
|
|
278
|
+
pose: currentOrbitPose(state),
|
|
279
|
+
});
|
|
280
|
+
if (replay.ok && replay.playId) {
|
|
281
|
+
state.orbitSinePlayId = replay.playId;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
})();
|
|
285
|
+
}, ORBIT_SINE_POLL_MS);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
async function startOrbitSinePlay(
|
|
289
|
+
sessionId: string,
|
|
290
|
+
state: SessionState,
|
|
291
|
+
): Promise<void> {
|
|
292
|
+
await stopOrbitSine(state);
|
|
293
|
+
const result = await play({
|
|
294
|
+
url: ORBIT_SINE_INLINE_URL,
|
|
295
|
+
bytes: buildOrbitSineInlineClipBase64(),
|
|
296
|
+
sessionIds: [sessionId],
|
|
297
|
+
volume: ORBIT_SINE_VOLUME,
|
|
298
|
+
pose: currentOrbitPose(state),
|
|
299
|
+
});
|
|
300
|
+
if (!result.ok || !result.playId) {
|
|
301
|
+
agentLog(
|
|
302
|
+
"warn",
|
|
303
|
+
`spatial-showcase orbit sine play failed: ${result.reason ?? "play_failed"}`,
|
|
304
|
+
sessionId,
|
|
305
|
+
);
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
state.orbitSinePlayId = result.playId;
|
|
309
|
+
startOrbitSineLoopPoll(sessionId, state);
|
|
310
|
+
}
|
|
311
|
+
|
|
190
312
|
function emitOrbitPose(sessionId: string, state: SessionState): void {
|
|
191
313
|
if (!state.orbitStartMs) {
|
|
192
314
|
return;
|
|
@@ -201,7 +323,8 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
|
|
|
201
323
|
state.lastOrbitPoseEmitMs = now;
|
|
202
324
|
const elapsedSec = (now - state.orbitStartMs) / 1000;
|
|
203
325
|
const pose = orbitTtsPose(elapsedSec, state.orbit);
|
|
204
|
-
const angleRad =
|
|
326
|
+
const angleRad =
|
|
327
|
+
(2 * Math.PI * elapsedSec) / (state.orbit.periodSec ?? 2 * Math.PI);
|
|
205
328
|
sendToClient(sessionId, {
|
|
206
329
|
type: "orbit_pose",
|
|
207
330
|
angleDeg: (angleRad * 180) / Math.PI,
|
|
@@ -213,16 +336,23 @@ function emitOrbitPose(sessionId: string, state: SessionState): void {
|
|
|
213
336
|
function startOrbitDemo(sessionId: string, state: SessionState): void {
|
|
214
337
|
stopOrbitInterval(state);
|
|
215
338
|
beginOrbitClock(state);
|
|
339
|
+
void startOrbitSinePlay(sessionId, state);
|
|
216
340
|
state.orbitTimer = setInterval(() => {
|
|
217
341
|
if (state.orbit.paused || !state.orbitStartMs) {
|
|
218
342
|
return;
|
|
219
343
|
}
|
|
220
|
-
const
|
|
344
|
+
const pose = currentOrbitPose(state);
|
|
345
|
+
if (state.orbitSinePlayId) {
|
|
346
|
+
void safeMixCall(sessionId, "setPlayPose", () =>
|
|
347
|
+
setPlayPose(state.orbitSinePlayId!, pose),
|
|
348
|
+
);
|
|
349
|
+
}
|
|
221
350
|
void safeMixCall(sessionId, "setTtsPose", () =>
|
|
222
|
-
setTtsPose(sessionId,
|
|
351
|
+
setTtsPose(sessionId, pose),
|
|
223
352
|
);
|
|
224
353
|
emitOrbitPose(sessionId, state);
|
|
225
354
|
}, ORBIT_TICK_MS);
|
|
355
|
+
emitOrbitPose(sessionId, state);
|
|
226
356
|
speak(sessionId, "I'll circle around you.");
|
|
227
357
|
}
|
|
228
358
|
|
|
@@ -250,7 +380,10 @@ function broadcastRoomStateIfChanged(): void {
|
|
|
250
380
|
}
|
|
251
381
|
}
|
|
252
382
|
|
|
253
|
-
function startProximityRoomBroadcast(
|
|
383
|
+
function startProximityRoomBroadcast(
|
|
384
|
+
sessionId: string,
|
|
385
|
+
state: SessionState,
|
|
386
|
+
): void {
|
|
254
387
|
clearRoomStateTimer(state);
|
|
255
388
|
state.roomStateTimer = setInterval(() => {
|
|
256
389
|
broadcastRoomStateIfChanged();
|
|
@@ -300,6 +433,15 @@ async function startLoopPadPoll(
|
|
|
300
433
|
clearLoopPad(sessionId, state, playId);
|
|
301
434
|
state.playIds.add(replay.playId);
|
|
302
435
|
state.loopPads.set(replay.playId, pad);
|
|
436
|
+
ack(sessionId, "pad", {
|
|
437
|
+
playId: replay.playId,
|
|
438
|
+
clipId: pad.clipId,
|
|
439
|
+
x: pad.x,
|
|
440
|
+
z: pad.z,
|
|
441
|
+
...(pad.placement !== undefined
|
|
442
|
+
? { placement: pad.placement }
|
|
443
|
+
: {}),
|
|
444
|
+
});
|
|
303
445
|
void startLoopPadPoll(sessionId, state, replay.playId, pad);
|
|
304
446
|
}
|
|
305
447
|
} else if (status.status) {
|
|
@@ -324,24 +466,27 @@ export async function handlePadPlay(
|
|
|
324
466
|
volume?: number,
|
|
325
467
|
placement?: PlayOptions["placement"],
|
|
326
468
|
): Promise<void> {
|
|
327
|
-
if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
|
|
328
|
-
ackError(sessionId, "pad", "busy");
|
|
329
|
-
return;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
469
|
const assetOrigin = state.assetOrigin;
|
|
333
470
|
if (!assetOrigin) {
|
|
334
|
-
ackError(sessionId, "pad", "missing_asset_origin");
|
|
471
|
+
ackError(sessionId, "pad", "missing_asset_origin", { clipId });
|
|
335
472
|
return;
|
|
336
473
|
}
|
|
337
474
|
|
|
338
475
|
const url = resolveClipUrl(assetOrigin, clipId);
|
|
339
476
|
if (!url) {
|
|
340
|
-
ackError(sessionId, "pad", "invalid_asset_origin");
|
|
477
|
+
ackError(sessionId, "pad", "invalid_asset_origin", { clipId });
|
|
341
478
|
return;
|
|
342
479
|
}
|
|
343
480
|
|
|
344
481
|
const sound = SHOWCASE_SOUND_FILES[clipId];
|
|
482
|
+
if (sound.loop) {
|
|
483
|
+
stopLoopPadsForClip(sessionId, state, clipId);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
if (state.playIds.size >= MAX_ACTIVE_PLAYS) {
|
|
487
|
+
ackError(sessionId, "pad", "busy", { clipId });
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
345
490
|
const playVolume = volume ?? sound.defaultVolume;
|
|
346
491
|
const options: PlayOptions = {
|
|
347
492
|
url,
|
|
@@ -352,12 +497,18 @@ export async function handlePadPlay(
|
|
|
352
497
|
|
|
353
498
|
const result = await play(options);
|
|
354
499
|
if (!result.ok || !result.playId) {
|
|
355
|
-
ackError(sessionId, "pad", result.reason ?? "play_failed");
|
|
500
|
+
ackError(sessionId, "pad", result.reason ?? "play_failed", { clipId });
|
|
356
501
|
return;
|
|
357
502
|
}
|
|
358
503
|
|
|
359
504
|
state.playIds.add(result.playId);
|
|
360
|
-
ack(sessionId, "pad", {
|
|
505
|
+
ack(sessionId, "pad", {
|
|
506
|
+
playId: result.playId,
|
|
507
|
+
clipId,
|
|
508
|
+
x,
|
|
509
|
+
z,
|
|
510
|
+
...(placement !== undefined ? { placement } : {}),
|
|
511
|
+
});
|
|
361
512
|
|
|
362
513
|
if (sound.loop) {
|
|
363
514
|
const pad: LoopPadState = {
|
|
@@ -394,9 +545,13 @@ async function handleJoin(
|
|
|
394
545
|
ack(sessionId, "join");
|
|
395
546
|
|
|
396
547
|
switch (demo) {
|
|
397
|
-
case "orbit":
|
|
548
|
+
case "orbit": {
|
|
549
|
+
void safeMixCall(sessionId, "setSttEnabled", () =>
|
|
550
|
+
setSttEnabled({ enabled: false, sessionId }),
|
|
551
|
+
);
|
|
398
552
|
startOrbitDemo(sessionId, state);
|
|
399
553
|
break;
|
|
554
|
+
}
|
|
400
555
|
case "soundboard":
|
|
401
556
|
break;
|
|
402
557
|
case "proximity": {
|
|
@@ -421,11 +576,11 @@ async function handleJoin(
|
|
|
421
576
|
return;
|
|
422
577
|
}
|
|
423
578
|
state.inProximityRoom = true;
|
|
579
|
+
startProximityRoomBroadcast(sessionId, state);
|
|
424
580
|
speak(sessionId, "Welcome to the proximity room.");
|
|
425
581
|
await safeMixCall(sessionId, "setTtsPose", () =>
|
|
426
582
|
setTtsPose(sessionId, poseAt(0, 0)),
|
|
427
583
|
);
|
|
428
|
-
startProximityRoomBroadcast(sessionId, state);
|
|
429
584
|
break;
|
|
430
585
|
}
|
|
431
586
|
}
|
|
@@ -439,8 +594,12 @@ function teardownSession(sessionId: string): void {
|
|
|
439
594
|
|
|
440
595
|
resetOrbitSession(state);
|
|
441
596
|
clearRoomStateTimer(state);
|
|
597
|
+
void stopOrbitSine(state);
|
|
442
598
|
clearAllPlays(sessionId, state);
|
|
443
599
|
void safeMixCall(sessionId, "clearTtsPose", () => clearTtsPose(sessionId));
|
|
600
|
+
void safeMixCall(sessionId, "setSttEnabled", () =>
|
|
601
|
+
setSttEnabled({ enabled: true, sessionId }),
|
|
602
|
+
);
|
|
444
603
|
|
|
445
604
|
if (state.inProximityRoom) {
|
|
446
605
|
proximityRoom.leave(sessionId);
|
|
@@ -512,6 +671,10 @@ async function dispatchMessage(
|
|
|
512
671
|
ackError(sessionId, "pad", "wrong_demo");
|
|
513
672
|
return;
|
|
514
673
|
}
|
|
674
|
+
if (!isShowcaseClipId(message.clipId)) {
|
|
675
|
+
ackError(sessionId, "pad", "invalid_clip_id");
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
515
678
|
await handlePadPlay(
|
|
516
679
|
sessionId,
|
|
517
680
|
state,
|
|
@@ -629,14 +792,6 @@ defineAgent({
|
|
|
629
792
|
});
|
|
630
793
|
},
|
|
631
794
|
|
|
632
|
-
onUserSpeechFinal({ sessionId, text }) {
|
|
633
|
-
const state = sessions.get(sessionId);
|
|
634
|
-
if (!state || state.demo !== "orbit") {
|
|
635
|
-
return;
|
|
636
|
-
}
|
|
637
|
-
speak(sessionId, `You said: ${text}`);
|
|
638
|
-
},
|
|
639
|
-
|
|
640
795
|
onDataChannelMessage(ctx) {
|
|
641
796
|
const message = parseShowcaseMessage(ctx.message);
|
|
642
797
|
if (!message) {
|
|
@@ -164,7 +164,9 @@ export function isShowcaseDemo(value: unknown): value is ShowcaseDemo {
|
|
|
164
164
|
return typeof value === "string" && SHOWCASE_DEMOS.has(value);
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
-
export function isShowcaseInbound(
|
|
167
|
+
export function isShowcaseInbound(
|
|
168
|
+
message: unknown,
|
|
169
|
+
): message is ShowcaseInbound {
|
|
168
170
|
return parseShowcaseMessage(message) !== null;
|
|
169
171
|
}
|
|
170
172
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/** Inline orbit sine clip — 16 kHz mono s16le, decoded size under 64 KiB play cap. */
|
|
2
|
+
|
|
3
|
+
/** 2 s keeps loop-restart gaps (250 ms poll) infrequent; 16 kHz mono still fits under 64 KiB. */
|
|
4
|
+
export const ORBIT_SINE_DURATION_MS = 2000;
|
|
5
|
+
export const ORBIT_SINE_FREQUENCY_HZ = 440;
|
|
6
|
+
export const ORBIT_SINE_AMPLITUDE = 8000;
|
|
7
|
+
export const ORBIT_SINE_INLINE_URL =
|
|
8
|
+
"https://example.com/inline-orbit-sine.wav";
|
|
9
|
+
|
|
10
|
+
/** Looping ~440 Hz sine WAV as base64 for {@link play} inline bytes. */
|
|
11
|
+
export function buildOrbitSineInlineClipBase64(
|
|
12
|
+
durationMs = ORBIT_SINE_DURATION_MS,
|
|
13
|
+
frequencyHz = ORBIT_SINE_FREQUENCY_HZ,
|
|
14
|
+
amplitude = ORBIT_SINE_AMPLITUDE,
|
|
15
|
+
): string {
|
|
16
|
+
const sampleRate = 16_000;
|
|
17
|
+
const channels = 1;
|
|
18
|
+
const bytesPerSample = 2;
|
|
19
|
+
const numSamples = Math.floor((sampleRate * durationMs) / 1000);
|
|
20
|
+
const dataSize = numSamples * channels * bytesPerSample;
|
|
21
|
+
const buffer = Buffer.alloc(44 + dataSize);
|
|
22
|
+
buffer.write("RIFF", 0);
|
|
23
|
+
buffer.writeUInt32LE(36 + dataSize, 4);
|
|
24
|
+
buffer.write("WAVE", 8);
|
|
25
|
+
buffer.write("fmt ", 12);
|
|
26
|
+
buffer.writeUInt32LE(16, 16);
|
|
27
|
+
buffer.writeUInt16LE(1, 20);
|
|
28
|
+
buffer.writeUInt16LE(channels, 22);
|
|
29
|
+
buffer.writeUInt32LE(sampleRate, 24);
|
|
30
|
+
buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
|
|
31
|
+
buffer.writeUInt16LE(channels * bytesPerSample, 32);
|
|
32
|
+
buffer.writeUInt16LE(16, 34);
|
|
33
|
+
buffer.write("data", 36);
|
|
34
|
+
buffer.writeUInt32LE(dataSize, 40);
|
|
35
|
+
for (let i = 0; i < numSamples; i++) {
|
|
36
|
+
const t = i / sampleRate;
|
|
37
|
+
const sample = Math.round(
|
|
38
|
+
amplitude * Math.sin(2 * Math.PI * frequencyHz * t),
|
|
39
|
+
);
|
|
40
|
+
buffer.writeInt16LE(sample, 44 + i * 2);
|
|
41
|
+
}
|
|
42
|
+
return buffer.toString("base64");
|
|
43
|
+
}
|