@voicethere/agent 0.5.6 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.js +63 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/protocol.d.ts +99 -2
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +4 -0
- package/dist/protocol.js.map +1 -1
- package/dist/runtime.d.ts +45 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +230 -0
- package/dist/runtime.js.map +1 -1
- package/dist/templates/echo/agent.js +61 -2
- package/dist/templates/echo-dc/agent.js +61 -2
- package/dist/templates/game-sync/agent.js +125 -33
- package/dist/templates/index.d.ts +2 -0
- package/dist/templates/index.d.ts.map +1 -1
- package/dist/templates/index.js +14 -0
- package/dist/templates/index.js.map +1 -1
- package/dist/templates/positional-tts/agent.js +810 -0
- package/dist/templates/recording-consent/agent.js +61 -2
- package/dist/templates/registry.d.ts.map +1 -1
- package/dist/templates/registry.js +17 -0
- package/dist/templates/registry.js.map +1 -1
- package/dist/templates/voice-showcase/agent.js +102 -21
- package/dist/templates/voice-starter/agent.js +61 -2
- package/dist/templates/webhooks/agent.js +61 -2
- package/dist/templates/webhooks-redis/agent.js +61 -2
- package/package.json +10 -4
- package/templates/README.md +12 -6
- package/templates/game-sync-world-layout.ts +45 -0
- package/templates/game-sync.ts +39 -24
- package/templates/mix-smoke.ts +224 -0
- package/templates/positional-tts/agent.ts +115 -0
- package/templates/positional-tts/orbit.ts +11 -0
- package/templates/voice-showcase/agent.ts +21 -22
- package/templates/voice-showcase/delivery.ts +57 -0
|
@@ -10891,6 +10891,18 @@ function isRecordingControlAckMessage(value) {
|
|
|
10891
10891
|
const msg = value;
|
|
10892
10892
|
return msg.type === "recording_control_ack" && typeof msg.requestId === "string";
|
|
10893
10893
|
}
|
|
10894
|
+
function isMixControlAckMessage(value) {
|
|
10895
|
+
if (!value || typeof value !== "object")
|
|
10896
|
+
return false;
|
|
10897
|
+
const msg = value;
|
|
10898
|
+
return msg.type === "mix_control_ack" && typeof msg.requestId === "string";
|
|
10899
|
+
}
|
|
10900
|
+
function isSttControlAckMessage(value) {
|
|
10901
|
+
if (!value || typeof value !== "object")
|
|
10902
|
+
return false;
|
|
10903
|
+
const msg = value;
|
|
10904
|
+
return msg.type === "stt_control_ack" && typeof msg.requestId === "string";
|
|
10905
|
+
}
|
|
10894
10906
|
function isWebhookMessage(value) {
|
|
10895
10907
|
if (!value || typeof value !== "object")
|
|
10896
10908
|
return false;
|
|
@@ -10933,7 +10945,7 @@ function isParentMessage(value) {
|
|
|
10933
10945
|
if (!value || typeof value !== "object")
|
|
10934
10946
|
return false;
|
|
10935
10947
|
const msg = value;
|
|
10936
|
-
return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "webhook";
|
|
10948
|
+
return msg.type === "session_start" || msg.type === "speech_event" || msg.type === "session_end" || msg.type === "data_channel_message" || msg.type === "data_channel_binary" || msg.type === "idle_timeout" || msg.type === "recording_control_ack" || msg.type === "mix_control_ack" || msg.type === "stt_control_ack" || msg.type === "webhook";
|
|
10937
10949
|
}
|
|
10938
10950
|
function isSessionScopedParentMessage(value) {
|
|
10939
10951
|
return isParentMessage(value) && !isWebhookMessage(value);
|
|
@@ -10947,8 +10959,12 @@ function parseDataChannelPayload(raw) {
|
|
|
10947
10959
|
}
|
|
10948
10960
|
var peerEnvBySessionId = /* @__PURE__ */ new Map();
|
|
10949
10961
|
var recordingAvailableBySessionId = /* @__PURE__ */ new Map();
|
|
10962
|
+
var mixAvailableBySessionId = /* @__PURE__ */ new Map();
|
|
10963
|
+
var ttsPoseAvailableBySessionId = /* @__PURE__ */ new Map();
|
|
10950
10964
|
var endedSessionIds = /* @__PURE__ */ new Set();
|
|
10951
10965
|
var pendingRecordingAcks = /* @__PURE__ */ new Map();
|
|
10966
|
+
var pendingMixAcks = /* @__PURE__ */ new Map();
|
|
10967
|
+
var pendingSttAcks = /* @__PURE__ */ new Map();
|
|
10952
10968
|
function handleRecordingControlAck(message) {
|
|
10953
10969
|
const pending = pendingRecordingAcks.get(message.requestId);
|
|
10954
10970
|
if (!pending)
|
|
@@ -10961,6 +10977,30 @@ function handleRecordingControlAck(message) {
|
|
|
10961
10977
|
requestId: message.requestId
|
|
10962
10978
|
});
|
|
10963
10979
|
}
|
|
10980
|
+
function handleMixControlAck(message) {
|
|
10981
|
+
const pending = pendingMixAcks.get(message.requestId);
|
|
10982
|
+
if (!pending)
|
|
10983
|
+
return;
|
|
10984
|
+
clearTimeout(pending.timer);
|
|
10985
|
+
pendingMixAcks.delete(message.requestId);
|
|
10986
|
+
pending.resolve({
|
|
10987
|
+
ok: message.ok,
|
|
10988
|
+
reason: message.reason,
|
|
10989
|
+
requestId: message.requestId
|
|
10990
|
+
});
|
|
10991
|
+
}
|
|
10992
|
+
function handleSttControlAck(message) {
|
|
10993
|
+
const pending = pendingSttAcks.get(message.requestId);
|
|
10994
|
+
if (!pending)
|
|
10995
|
+
return;
|
|
10996
|
+
clearTimeout(pending.timer);
|
|
10997
|
+
pendingSttAcks.delete(message.requestId);
|
|
10998
|
+
pending.resolve({
|
|
10999
|
+
ok: message.ok,
|
|
11000
|
+
reason: message.reason,
|
|
11001
|
+
requestId: message.requestId
|
|
11002
|
+
});
|
|
11003
|
+
}
|
|
10964
11004
|
function clearPendingRecordingAcksForSession(sessionId, reason) {
|
|
10965
11005
|
for (const [requestId, pending] of pendingRecordingAcks) {
|
|
10966
11006
|
if (pending.sessionId !== sessionId)
|
|
@@ -11016,6 +11056,11 @@ function allowOutboundForSession(sessionId) {
|
|
|
11016
11056
|
return queue.isLive(sessionId);
|
|
11017
11057
|
}
|
|
11018
11058
|
function sendParentMessage(message) {
|
|
11059
|
+
const msgType = message && typeof message === "object" && "type" in message ? message.type : void 0;
|
|
11060
|
+
if (msgType === "mix_control" || msgType === "stt_control") {
|
|
11061
|
+
process.send?.(message);
|
|
11062
|
+
return;
|
|
11063
|
+
}
|
|
11019
11064
|
const sessionId = message && typeof message === "object" && "sessionId" in message && typeof message.sessionId === "string" ? message.sessionId : void 0;
|
|
11020
11065
|
if (!allowOutboundForSession(sessionId))
|
|
11021
11066
|
return;
|
|
@@ -11097,6 +11142,8 @@ async function handleParentMessage(message, handlers) {
|
|
|
11097
11142
|
endedSessionIds.delete(message.sessionId);
|
|
11098
11143
|
peerEnvBySessionId.set(message.sessionId, message.env);
|
|
11099
11144
|
recordingAvailableBySessionId.set(message.sessionId, message.recordingAvailable ?? false);
|
|
11145
|
+
mixAvailableBySessionId.set(message.sessionId, message.mixAvailable ?? false);
|
|
11146
|
+
ttsPoseAvailableBySessionId.set(message.sessionId, message.ttsPoseAvailable ?? false);
|
|
11100
11147
|
const sessionStartInitDelayMs = resolveSessionStartInitDelayMs();
|
|
11101
11148
|
if (sessionStartInitDelayMs > 0) {
|
|
11102
11149
|
await new Promise((resolve) => setTimeout(resolve, sessionStartInitDelayMs));
|
|
@@ -11104,7 +11151,9 @@ async function handleParentMessage(message, handlers) {
|
|
|
11104
11151
|
await (handlers.onClientJoin ?? handlers.onSessionStart)?.({
|
|
11105
11152
|
sessionId: message.sessionId,
|
|
11106
11153
|
env: message.env,
|
|
11107
|
-
recordingAvailable: message.recordingAvailable ?? false
|
|
11154
|
+
recordingAvailable: message.recordingAvailable ?? false,
|
|
11155
|
+
mixAvailable: message.mixAvailable ?? false,
|
|
11156
|
+
ttsPoseAvailable: message.ttsPoseAvailable ?? false
|
|
11108
11157
|
});
|
|
11109
11158
|
sendParentMessage({
|
|
11110
11159
|
type: "session_start_ack",
|
|
@@ -11142,6 +11191,8 @@ async function handleParentMessage(message, handlers) {
|
|
|
11142
11191
|
clearPendingRecordingAcksForSession(message.sessionId, "session_ended");
|
|
11143
11192
|
peerEnvBySessionId.delete(message.sessionId);
|
|
11144
11193
|
recordingAvailableBySessionId.delete(message.sessionId);
|
|
11194
|
+
mixAvailableBySessionId.delete(message.sessionId);
|
|
11195
|
+
ttsPoseAvailableBySessionId.delete(message.sessionId);
|
|
11145
11196
|
await (handlers.onClientLeave ?? handlers.onSessionEnd)?.({
|
|
11146
11197
|
sessionId: message.sessionId
|
|
11147
11198
|
});
|
|
@@ -11161,6 +11212,14 @@ function defineAgent(handlers) {
|
|
|
11161
11212
|
handleRecordingControlAck(message);
|
|
11162
11213
|
return;
|
|
11163
11214
|
}
|
|
11215
|
+
if (isMixControlAckMessage(message)) {
|
|
11216
|
+
handleMixControlAck(message);
|
|
11217
|
+
return;
|
|
11218
|
+
}
|
|
11219
|
+
if (isSttControlAckMessage(message)) {
|
|
11220
|
+
handleSttControlAck(message);
|
|
11221
|
+
return;
|
|
11222
|
+
}
|
|
11164
11223
|
if (isWebhookMessage(message)) {
|
|
11165
11224
|
void agentStartReady.then(() => handleWebhookMessage(message, handlers));
|
|
11166
11225
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voicethere/agent",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "VoiceThere customer agent SDK — IPC types and runtime helpers for sandboxed child bundles",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -28,6 +28,12 @@
|
|
|
28
28
|
"require": "./dist/templates/index.js",
|
|
29
29
|
"default": "./dist/templates/index.js"
|
|
30
30
|
},
|
|
31
|
+
"./build": {
|
|
32
|
+
"types": "./dist/build-bundle.d.ts",
|
|
33
|
+
"import": "./dist/build-bundle.js",
|
|
34
|
+
"require": "./dist/build-bundle.js",
|
|
35
|
+
"default": "./dist/build-bundle.js"
|
|
36
|
+
},
|
|
31
37
|
"./templates/redis-sync/world-layout": {
|
|
32
38
|
"types": "./templates/redis-sync/world-layout.ts",
|
|
33
39
|
"import": "./templates/redis-sync/world-layout.ts",
|
|
@@ -89,9 +95,9 @@
|
|
|
89
95
|
"@node-webrtc-rust/sdk": ">=0.6.5"
|
|
90
96
|
},
|
|
91
97
|
"devDependencies": {
|
|
92
|
-
"@node-webrtc-rust/helpers": "0.
|
|
93
|
-
"@node-webrtc-rust/sdk": "0.
|
|
94
|
-
"@node-webrtc-rust/signaling": "0.
|
|
98
|
+
"@node-webrtc-rust/helpers": "0.8.0",
|
|
99
|
+
"@node-webrtc-rust/sdk": "0.8.0",
|
|
100
|
+
"@node-webrtc-rust/signaling": "0.8.0",
|
|
95
101
|
"@types/node": "22.20.1",
|
|
96
102
|
"http-server": "14.1.1",
|
|
97
103
|
"ioredis": "5.11.1",
|
package/templates/README.md
CHANGED
|
@@ -15,10 +15,10 @@ import {
|
|
|
15
15
|
|
|
16
16
|
## Product vs e2e
|
|
17
17
|
|
|
18
|
-
| Kind | Dashboard create
|
|
19
|
-
| ----------- |
|
|
20
|
-
| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
|
|
21
|
-
| **e2e** | No
|
|
18
|
+
| Kind | Dashboard create | Prebuilt seed bundle | Typical consumer |
|
|
19
|
+
| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ----------------------- |
|
|
20
|
+
| **product** | Yes (`echo`, `echo-dc`, `voice-starter`, `game-sync`, `voice-showcase`, `recording-consent`, `positional-tts`, `webhooks`, `webhooks-redis`) | Yes — `dist/templates/<id>/agent.js` | Platform project create |
|
|
21
|
+
| **e2e** | No | No — build from sources at test time | `voicethere/e2e` smokes |
|
|
22
22
|
|
|
23
23
|
Product templates always set `seedOnCreate: true`. CI fails if a product template is missing its prebuilt bundle after `npm run build`.
|
|
24
24
|
|
|
@@ -63,9 +63,9 @@ When `AGENT_REDIS_URL` is set (project Redis), the world blob is stored at `game
|
|
|
63
63
|
|
|
64
64
|
### `voice-showcase/` (`voice-showcase`)
|
|
65
65
|
|
|
66
|
-
Conversational voice demo for landing and dashboard previews — greets the user, asks for a name, then offers a menu: weather (Open-Meteo, no API key), count 1–10, short recipes, and rotating fun facts. Typed chat and voice finals share the same handler. Sends structured `menu` payloads plus `chat_reply` for the chat log.
|
|
66
|
+
Conversational voice demo for landing and dashboard previews — greets the user, asks for a name, then offers a menu: weather (Open-Meteo, no API key), count 1–10, short recipes, and rotating fun facts. Typed chat and voice finals share the same handler. Sends structured `menu` payloads plus `chat_reply` for the chat log, then triggers TTS play so the client has the spoken text before audio starts.
|
|
67
67
|
|
|
68
|
-
Sources: `voice-showcase/agent.ts` (defineAgent wiring), `conversation.ts` (pure state machine), `weather.ts`, `recipes.ts`, `fun-facts.ts`.
|
|
68
|
+
Sources: `voice-showcase/agent.ts` (defineAgent wiring), `conversation.ts` (pure state machine), `delivery.ts` (send-then-play order), `weather.ts`, `recipes.ts`, `fun-facts.ts`.
|
|
69
69
|
|
|
70
70
|
### `recording-consent/` (`recording-consent`)
|
|
71
71
|
|
|
@@ -73,6 +73,12 @@ Demonstrates conversation recording consent on connect: asks whether recording i
|
|
|
73
73
|
|
|
74
74
|
Sources: `recording-consent/agent.ts` (defineAgent wiring), `conversation.ts` (pure state machine).
|
|
75
75
|
|
|
76
|
+
### `positional-tts/` (`positional-tts`)
|
|
77
|
+
|
|
78
|
+
Voice+Data demo — enables positional mixing and orbits each listener’s TTS speaker with `setTtsPose` on a ~50 ms timer. Speaks a short greeting on connect and echoes voice finals / chat so you hear panning while TTS plays. Requires Voice+Data runner mode (`isMixAvailable`).
|
|
79
|
+
|
|
80
|
+
Sources: `positional-tts/agent.ts` (defineAgent wiring), `positional-tts/orbit.ts` (pure circle helper for tests).
|
|
81
|
+
|
|
76
82
|
### `webhooks.ts` (`webhooks`)
|
|
77
83
|
|
|
78
84
|
Inbound HTTP webhook sample — verifies `x-agent-webhook-signature` HMAC on the **raw body** with `AGENT_WEBHOOK_SIGNING_SECRET`, then `JSON.parse` and fans out to connected sessions via DataChannel + `speak`.
|
|
@@ -162,6 +162,51 @@ export function collectActiveObjectIds(world: Float32Array): number[] {
|
|
|
162
162
|
return ids;
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
export interface LiveWorldObjectInfo {
|
|
166
|
+
objectId: number;
|
|
167
|
+
ownerSessionId: string;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Snapshot occupancy from live world slots; ownerSessionId from map or empty string. */
|
|
171
|
+
export function liveWorldSnapshot(
|
|
172
|
+
world: Float32Array,
|
|
173
|
+
owners: ReadonlyMap<number, string>,
|
|
174
|
+
): LiveWorldObjectInfo[] {
|
|
175
|
+
const ids = collectActiveObjectIds(world);
|
|
176
|
+
ids.sort((a, b) => a - b);
|
|
177
|
+
return ids.map((objectId) => ({
|
|
178
|
+
objectId,
|
|
179
|
+
ownerSessionId: owners.get(objectId) ?? "",
|
|
180
|
+
}));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function planRedisSimTick(options: {
|
|
184
|
+
lockAcquired: boolean;
|
|
185
|
+
connectedSessions: ReadonlySet<string>;
|
|
186
|
+
}): "simulate" | "relay" | "noop" {
|
|
187
|
+
if (options.lockAcquired) {
|
|
188
|
+
return "simulate";
|
|
189
|
+
}
|
|
190
|
+
if (options.connectedSessions.size > 0) {
|
|
191
|
+
return "relay";
|
|
192
|
+
}
|
|
193
|
+
return "noop";
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Finite positive elapsed seconds, clamped to [1/fallbackHz, 0.05]. */
|
|
197
|
+
export function clampSimulationDtSec(
|
|
198
|
+
elapsedMs: number,
|
|
199
|
+
fallbackHz: number,
|
|
200
|
+
): number {
|
|
201
|
+
const minDt = 1 / fallbackHz;
|
|
202
|
+
const maxDt = 0.05;
|
|
203
|
+
if (!Number.isFinite(elapsedMs) || elapsedMs <= 0) {
|
|
204
|
+
return minDt;
|
|
205
|
+
}
|
|
206
|
+
const elapsedSec = elapsedMs / 1000;
|
|
207
|
+
return Math.min(maxDt, Math.max(minDt, elapsedSec));
|
|
208
|
+
}
|
|
209
|
+
|
|
165
210
|
/**
|
|
166
211
|
* Decode a Redis / Node Buffer into a fixed-size world Float32Array.
|
|
167
212
|
*/
|
package/templates/game-sync.ts
CHANGED
|
@@ -61,15 +61,17 @@ import {
|
|
|
61
61
|
simulateWorldStep,
|
|
62
62
|
} from "./game-sync-sim.js";
|
|
63
63
|
import {
|
|
64
|
+
clampSimulationDtSec,
|
|
64
65
|
collectActiveObjectIds,
|
|
65
|
-
commitSimulatedWorld,
|
|
66
66
|
countLiveObjects,
|
|
67
67
|
createEmptyWorldBuffer,
|
|
68
68
|
findFirstEmptySlot,
|
|
69
|
+
liveWorldSnapshot,
|
|
69
70
|
markSlotFree,
|
|
70
71
|
normalizeWorldBuffer,
|
|
71
72
|
objectIdToSlot,
|
|
72
73
|
OBJECT_SLOT_BYTE_LENGTH,
|
|
74
|
+
planRedisSimTick,
|
|
73
75
|
REDIS_SIM_LOCK_KEY,
|
|
74
76
|
REDIS_WORLD_KEY,
|
|
75
77
|
slotToObjectId,
|
|
@@ -92,6 +94,7 @@ let worldState = createEmptyWorldBuffer();
|
|
|
92
94
|
let redis: Redis | null = null;
|
|
93
95
|
let broadcastTimer: NodeJS.Timeout | null = null;
|
|
94
96
|
let worldMutationChain: Promise<void> = Promise.resolve();
|
|
97
|
+
let lastTickTimeMs = 0;
|
|
95
98
|
|
|
96
99
|
function withWorldMutation<T>(fn: () => Promise<T>): Promise<T> {
|
|
97
100
|
const run = worldMutationChain.then(fn);
|
|
@@ -140,11 +143,6 @@ async function withRedisSimLock<T>(
|
|
|
140
143
|
}
|
|
141
144
|
}
|
|
142
145
|
|
|
143
|
-
interface TrackedObjectInfo {
|
|
144
|
-
objectId: number;
|
|
145
|
-
ownerSessionId: string;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
146
|
function rand(min: number, max: number): number {
|
|
149
147
|
return min + Math.random() * (max - min);
|
|
150
148
|
}
|
|
@@ -315,13 +313,11 @@ async function unregisterObject(
|
|
|
315
313
|
return { ok: true, objectId: target.objectId };
|
|
316
314
|
}
|
|
317
315
|
|
|
318
|
-
function
|
|
319
|
-
const objects
|
|
320
|
-
for (const
|
|
321
|
-
|
|
316
|
+
function broadcastWorldSnapshot(): void {
|
|
317
|
+
const objects = liveWorldSnapshot(worldState, objectOwners);
|
|
318
|
+
for (const sessionId of connectedSessions) {
|
|
319
|
+
sendToClient(sessionId, { type: "world_snapshot", objects });
|
|
322
320
|
}
|
|
323
|
-
objects.sort((a, b) => a.objectId - b.objectId);
|
|
324
|
-
return objects;
|
|
325
321
|
}
|
|
326
322
|
|
|
327
323
|
function notifyObjectRegistered(
|
|
@@ -375,33 +371,47 @@ async function saveWorldToRedis(world: Float32Array): Promise<void> {
|
|
|
375
371
|
}
|
|
376
372
|
|
|
377
373
|
async function runSimulationTick(): Promise<void> {
|
|
374
|
+
const now = Date.now();
|
|
375
|
+
const dt = clampSimulationDtSec(
|
|
376
|
+
lastTickTimeMs === 0 ? 0 : now - lastTickTimeMs,
|
|
377
|
+
BROADCAST_HZ,
|
|
378
|
+
);
|
|
379
|
+
lastTickTimeMs = now;
|
|
380
|
+
|
|
378
381
|
if (!redis) {
|
|
379
382
|
await withWorldMutation(async () => {
|
|
380
|
-
|
|
381
|
-
simulateWorldStep(worldState, 1 / BROADCAST_HZ, activeObjectIds);
|
|
383
|
+
simulateWorldStep(worldState, dt, collectActiveObjectIds(worldState));
|
|
382
384
|
});
|
|
383
385
|
broadcastWorldBuffer(worldState);
|
|
384
386
|
return;
|
|
385
387
|
}
|
|
386
388
|
|
|
387
389
|
await withWorldMutation(async () => {
|
|
388
|
-
await withRedisSimLock(
|
|
390
|
+
const lockResult = await withRedisSimLock(
|
|
389
391
|
async () => {
|
|
390
392
|
const world = await loadWorldFromRedis();
|
|
391
393
|
const activeObjectIds = collectActiveObjectIds(world);
|
|
392
|
-
simulateWorldStep(world,
|
|
393
|
-
const latestRedis = await loadWorldFromRedis();
|
|
394
|
-
commitSimulatedWorld(world, latestRedis);
|
|
394
|
+
simulateWorldStep(world, dt, activeObjectIds);
|
|
395
395
|
await saveWorldToRedis(world);
|
|
396
396
|
worldState = world;
|
|
397
|
+
broadcastWorldBuffer(world);
|
|
398
|
+
return true;
|
|
397
399
|
},
|
|
398
400
|
{ retryUntilAcquired: false },
|
|
399
401
|
);
|
|
400
|
-
});
|
|
401
402
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
403
|
+
if (lockResult === null) {
|
|
404
|
+
const plan = planRedisSimTick({
|
|
405
|
+
lockAcquired: false,
|
|
406
|
+
connectedSessions,
|
|
407
|
+
});
|
|
408
|
+
if (plan === "relay") {
|
|
409
|
+
const world = await loadWorldFromRedis();
|
|
410
|
+
worldState = world;
|
|
411
|
+
broadcastWorldBuffer(world);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
});
|
|
405
415
|
}
|
|
406
416
|
|
|
407
417
|
function startBroadcastLoopIfNeeded(): void {
|
|
@@ -466,12 +476,15 @@ defineAgent({
|
|
|
466
476
|
agentLog("info", "game-sync agent connected to project Redis world buffer");
|
|
467
477
|
},
|
|
468
478
|
|
|
469
|
-
onClientJoin({ sessionId }) {
|
|
479
|
+
async onClientJoin({ sessionId }) {
|
|
470
480
|
connectedSessions.add(sessionId);
|
|
471
481
|
startBroadcastLoopIfNeeded();
|
|
482
|
+
if (redis) {
|
|
483
|
+
worldState = await loadWorldFromRedis();
|
|
484
|
+
}
|
|
472
485
|
sendToClient(sessionId, {
|
|
473
486
|
type: "world_snapshot",
|
|
474
|
-
objects:
|
|
487
|
+
objects: liveWorldSnapshot(worldState, objectOwners),
|
|
475
488
|
});
|
|
476
489
|
agentLog("info", `join ${sessionId} connected=${connectedSessions.size}`);
|
|
477
490
|
},
|
|
@@ -516,6 +529,7 @@ defineAgent({
|
|
|
516
529
|
}
|
|
517
530
|
sendToClient(ctx.sessionId, { type: "register_ack", objectId });
|
|
518
531
|
notifyObjectRegistered(objectId, ctx.sessionId);
|
|
532
|
+
broadcastWorldSnapshot();
|
|
519
533
|
agentLog(
|
|
520
534
|
"info",
|
|
521
535
|
`register session=${ctx.sessionId} objectId=${objectId}`,
|
|
@@ -541,6 +555,7 @@ defineAgent({
|
|
|
541
555
|
type: "unregister_ack",
|
|
542
556
|
objectId: result.objectId,
|
|
543
557
|
});
|
|
558
|
+
broadcastWorldSnapshot();
|
|
544
559
|
agentLog(
|
|
545
560
|
"info",
|
|
546
561
|
`unregister_ack session=${ctx.sessionId} objectId=${result.objectId}`,
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* E2E agent for voice-data-mix-smoke — positional mix via data-channel commands.
|
|
3
|
+
*
|
|
4
|
+
* Protocol (JSON on control DC):
|
|
5
|
+
* - `{ type: "mix", action: "whoami" }`
|
|
6
|
+
* - `{ type: "mix", action: "create_group", groupId, clientIds }`
|
|
7
|
+
* - `{ type: "mix", action: "set_pose", clientId, pose }`
|
|
8
|
+
* - `{ type: "mix", action: "set_positional", enabled }`
|
|
9
|
+
* - `{ type: "mix", action: "list_clients" }` (optional)
|
|
10
|
+
*
|
|
11
|
+
* Acks: `{ type: "mix_ack", action, ok: true, ... }` or `{ ok: false, error }`.
|
|
12
|
+
* Ignores `ping` / chat strings (voice-control readiness). No TTS during smoke.
|
|
13
|
+
*/
|
|
14
|
+
import {
|
|
15
|
+
MIX_REQUIRES_VOICE_PLUS_DATA,
|
|
16
|
+
createMixGroup,
|
|
17
|
+
defineAgent,
|
|
18
|
+
sendToClient,
|
|
19
|
+
setClientPose,
|
|
20
|
+
setPositionalMixing,
|
|
21
|
+
} from "@voicethere/agent";
|
|
22
|
+
|
|
23
|
+
/** E2E fixture marker — rewritten before each fixture upload when needed. */
|
|
24
|
+
export const FIXTURE_MARKER = "mix-smoke-fixture-a";
|
|
25
|
+
|
|
26
|
+
export type MixPose = {
|
|
27
|
+
position: { x: number; y: number; z: number };
|
|
28
|
+
orientation: { x: number; y: number; z: number; w: number };
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export type MixCommand =
|
|
32
|
+
| { type: "mix"; action: "whoami" }
|
|
33
|
+
| {
|
|
34
|
+
type: "mix";
|
|
35
|
+
action: "create_group";
|
|
36
|
+
groupId: string;
|
|
37
|
+
clientIds: string[];
|
|
38
|
+
}
|
|
39
|
+
| { type: "mix"; action: "set_pose"; clientId: string; pose: MixPose }
|
|
40
|
+
| { type: "mix"; action: "set_positional"; enabled: boolean }
|
|
41
|
+
| { type: "mix"; action: "list_clients" };
|
|
42
|
+
|
|
43
|
+
export type MixAck =
|
|
44
|
+
| {
|
|
45
|
+
type: "mix_ack";
|
|
46
|
+
action: string;
|
|
47
|
+
ok: true;
|
|
48
|
+
sessionId?: string;
|
|
49
|
+
clientIds?: string[];
|
|
50
|
+
}
|
|
51
|
+
| { type: "mix_ack"; action: string; ok: false; error: string };
|
|
52
|
+
|
|
53
|
+
const connectedSessions = new Set<string>();
|
|
54
|
+
let mixAvailableForChild = false;
|
|
55
|
+
|
|
56
|
+
export function isMixCommand(message: unknown): message is MixCommand {
|
|
57
|
+
if (!message || typeof message !== "object") return false;
|
|
58
|
+
const record = message as { type?: unknown; action?: unknown };
|
|
59
|
+
if (record.type !== "mix" || typeof record.action !== "string") {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
switch (record.action) {
|
|
63
|
+
case "whoami":
|
|
64
|
+
case "list_clients":
|
|
65
|
+
return true;
|
|
66
|
+
case "create_group": {
|
|
67
|
+
const group = message as {
|
|
68
|
+
groupId?: unknown;
|
|
69
|
+
clientIds?: unknown;
|
|
70
|
+
};
|
|
71
|
+
return (
|
|
72
|
+
typeof group.groupId === "string" &&
|
|
73
|
+
group.groupId.trim().length > 0 &&
|
|
74
|
+
Array.isArray(group.clientIds) &&
|
|
75
|
+
group.clientIds.every((id) => typeof id === "string")
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
case "set_pose": {
|
|
79
|
+
const poseMsg = message as { clientId?: unknown; pose?: unknown };
|
|
80
|
+
return (
|
|
81
|
+
typeof poseMsg.clientId === "string" &&
|
|
82
|
+
poseMsg.clientId.trim().length > 0 &&
|
|
83
|
+
isMixPose(poseMsg.pose)
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
case "set_positional": {
|
|
87
|
+
const positional = message as { enabled?: unknown };
|
|
88
|
+
return typeof positional.enabled === "boolean";
|
|
89
|
+
}
|
|
90
|
+
default:
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function isMixPose(value: unknown): value is MixPose {
|
|
96
|
+
if (!value || typeof value !== "object") return false;
|
|
97
|
+
const pose = value as MixPose;
|
|
98
|
+
return isVec3(pose.position) && isQuat(pose.orientation);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function isVec3(value: unknown): value is { x: number; y: number; z: number } {
|
|
102
|
+
if (!value || typeof value !== "object") return false;
|
|
103
|
+
const v = value as { x?: unknown; y?: unknown; z?: unknown };
|
|
104
|
+
return (
|
|
105
|
+
typeof v.x === "number" &&
|
|
106
|
+
typeof v.y === "number" &&
|
|
107
|
+
typeof v.z === "number"
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function isQuat(
|
|
112
|
+
value: unknown,
|
|
113
|
+
): value is { x: number; y: number; z: number; w: number } {
|
|
114
|
+
if (!value || typeof value !== "object") return false;
|
|
115
|
+
const q = value as { x?: unknown; y?: unknown; z?: unknown; w?: unknown };
|
|
116
|
+
return (
|
|
117
|
+
typeof q.x === "number" &&
|
|
118
|
+
typeof q.y === "number" &&
|
|
119
|
+
typeof q.z === "number" &&
|
|
120
|
+
typeof q.w === "number"
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function ackOk(
|
|
125
|
+
sessionId: string,
|
|
126
|
+
action: string,
|
|
127
|
+
extra?: { sessionId?: string; clientIds?: string[] },
|
|
128
|
+
): void {
|
|
129
|
+
sendToClient(sessionId, {
|
|
130
|
+
type: "mix_ack",
|
|
131
|
+
action,
|
|
132
|
+
ok: true,
|
|
133
|
+
...extra,
|
|
134
|
+
} satisfies MixAck);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function ackError(sessionId: string, action: string, error: string): void {
|
|
138
|
+
sendToClient(sessionId, {
|
|
139
|
+
type: "mix_ack",
|
|
140
|
+
action,
|
|
141
|
+
ok: false,
|
|
142
|
+
error,
|
|
143
|
+
} satisfies MixAck);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function requireMix(sessionId: string, action: string): boolean {
|
|
147
|
+
if (mixAvailableForChild) {
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
ackError(sessionId, action, MIX_REQUIRES_VOICE_PLUS_DATA);
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async function handleMixCommand(
|
|
155
|
+
sessionId: string,
|
|
156
|
+
command: MixCommand,
|
|
157
|
+
): Promise<void> {
|
|
158
|
+
const { action } = command;
|
|
159
|
+
|
|
160
|
+
switch (action) {
|
|
161
|
+
case "whoami":
|
|
162
|
+
ackOk(sessionId, action, { sessionId });
|
|
163
|
+
return;
|
|
164
|
+
case "list_clients":
|
|
165
|
+
ackOk(sessionId, action, {
|
|
166
|
+
clientIds: [...connectedSessions],
|
|
167
|
+
});
|
|
168
|
+
return;
|
|
169
|
+
case "create_group": {
|
|
170
|
+
if (!requireMix(sessionId, action)) return;
|
|
171
|
+
const result = await createMixGroup({
|
|
172
|
+
id: command.groupId,
|
|
173
|
+
clientIds: command.clientIds,
|
|
174
|
+
});
|
|
175
|
+
if (!result.ok) {
|
|
176
|
+
ackError(sessionId, action, result.reason ?? "create_group failed");
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
ackOk(sessionId, action);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
case "set_pose": {
|
|
183
|
+
if (!requireMix(sessionId, action)) return;
|
|
184
|
+
const result = await setClientPose(command.clientId, command.pose);
|
|
185
|
+
if (!result.ok) {
|
|
186
|
+
ackError(sessionId, action, result.reason ?? "set_pose failed");
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
ackOk(sessionId, action);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
case "set_positional": {
|
|
193
|
+
if (!requireMix(sessionId, action)) return;
|
|
194
|
+
const result = await setPositionalMixing(command.enabled);
|
|
195
|
+
if (!result.ok) {
|
|
196
|
+
ackError(sessionId, action, result.reason ?? "set_positional failed");
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
ackOk(sessionId, action);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
default:
|
|
203
|
+
ackError(sessionId, "unknown", "unsupported mix action");
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
defineAgent({
|
|
208
|
+
onClientJoin(ctx) {
|
|
209
|
+
mixAvailableForChild = ctx.mixAvailable;
|
|
210
|
+
connectedSessions.add(ctx.sessionId);
|
|
211
|
+
},
|
|
212
|
+
|
|
213
|
+
onClientLeave({ sessionId }) {
|
|
214
|
+
connectedSessions.delete(sessionId);
|
|
215
|
+
},
|
|
216
|
+
|
|
217
|
+
onDataChannelMessage(ctx) {
|
|
218
|
+
if (isMixCommand(ctx.message)) {
|
|
219
|
+
void handleMixCommand(ctx.sessionId, ctx.message);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
// Ignore ping / chat — voice-control readiness only.
|
|
223
|
+
},
|
|
224
|
+
});
|