@unoverse-platform/studio 0.1.101 → 0.1.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local/sample-identity/brand.yaml +2 -0
- package/package.json +1 -1
- package/screens/TemplatesLibraryView.tsx +58 -98
- package/screens/TemplatesView.tsx +16 -0
- package/vendor/base/src/definitions/load/assets.ts +65 -0
- package/vendor/base/src/definitions/load/compile/component.ts +1 -0
- package/vendor/base/src/definitions/load/compile/index.ts +1 -1
- package/vendor/base/src/definitions/load/compile/sections.ts +26 -116
- package/vendor/base/src/definitions/load/compile/template.ts +41 -25
- package/vendor/base/src/definitions/load/expand/walk.ts +4 -1
- package/vendor/base/src/definitions/load/load.ts +6 -0
- package/vendor/base/src/definitions/model.ts +20 -41
- package/vendor/base/src/definitions/registry/homes.ts +9 -1
- package/vendor/base/src/lint/design/file.mjs +29 -5
- package/vendor/base/src/lint/design/vocabulary.mjs +1 -1
- package/vendor/base/src/lint/design/walk.mjs +31 -156
- package/vendor/base/src/manifests/executor/context.ts +1 -1
- package/vendor/base/src/manifests/executor/toolBridge.ts +3 -29
- package/vendor/base/src/manifests/runtime/duplex/audioLane.ts +35 -12
- package/vendor/base/src/manifests/runtime/duplex/session.ts +123 -48
- package/vendor/base/src/manifests/runtime/index.ts +1 -1
- package/vendor/base/src/platform/audioManager.ts +89 -14
- package/vendor/base/src/platform/clientTransport.ts +8 -0
- package/vendor/base/src/security/permissions.ts +2 -2
- package/vendor/sdk/appEngine.tsx +104 -46
- package/vendor/sdk/lib/UnoverseComponent.tsx +58 -5
- package/vendor/sdk/lib/app.tsx +9 -5
- package/vendor/sdk/lib/connection.tsx +14 -3
- package/vendor/sdk/lib/core/actions.ts +5 -2
- package/vendor/sdk/lib/index.ts +2 -2
- package/vendor/sdk/lib/primitives.tsx +38 -4
- package/vendor/sdk/lib/realtime/useAudioPlayback.ts +6 -1
- package/vendor/sdk/lib/voice.tsx +96 -13
- package/vendor/sdk/pageInterfaces.tsx +383 -0
package/vendor/sdk/lib/voice.tsx
CHANGED
|
@@ -77,10 +77,30 @@ export interface VoiceServiceState {
|
|
|
77
77
|
error: string | null;
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
+
export interface StartCall {
|
|
81
|
+
message?: string;
|
|
82
|
+
metadata?: Record<string, unknown>;
|
|
83
|
+
workflowId?: string;
|
|
84
|
+
targetTriggerNode?: string;
|
|
85
|
+
}
|
|
86
|
+
|
|
80
87
|
export interface VoiceService {
|
|
81
88
|
/** Spread into the template's data scope (channel overrides → rootData). */
|
|
82
89
|
state: VoiceServiceState;
|
|
83
|
-
|
|
90
|
+
/** Start a call. `start` is what it starts with, as any request to the workflow: the text,
|
|
91
|
+
* and metadata a trigger reads (`metadata.task` names the task). Omitted → "Start call".
|
|
92
|
+
* `workflowId`/`targetTriggerNode` name the workflow to run, else the session's own: a page
|
|
93
|
+
* template's binding (UNOVERSE_HOST_PAGE.md rule 4). Started while a call is live, it MOVES
|
|
94
|
+
* the call: the same line, the new workflow, and the server closes the session before it
|
|
95
|
+
* (VOICE_STREAMING_GUIDE.md, "One Live Session Per Conversation"). */
|
|
96
|
+
startCall: (start?: StartCall) => Promise<void>;
|
|
97
|
+
/** Join a call already running: the audio lane and the microphone for a workflow something
|
|
98
|
+
* else started (a canvas Run, a button, `JOIN_CALL`), without starting it again. `workflowId`
|
|
99
|
+
* is what the hang-up names; omitted, the session's own. Joining a call already joined is
|
|
100
|
+
* nothing. */
|
|
101
|
+
joinCall: (workflowId?: string) => Promise<void>;
|
|
102
|
+
/** Wake the speaker. Call it inside the click that may start a call. */
|
|
103
|
+
unlock: () => void;
|
|
84
104
|
endCall: () => Promise<void>;
|
|
85
105
|
toggleMute: () => void;
|
|
86
106
|
}
|
|
@@ -113,6 +133,11 @@ export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
|
|
|
113
133
|
const [lookupTool, setLookupTool] = useState<string | null>(null);
|
|
114
134
|
|
|
115
135
|
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
136
|
+
/** The workflow the live call is on, which a hang-up names; null with no call. */
|
|
137
|
+
const callWorkflowRef = useRef<string | null>(null);
|
|
138
|
+
/** Lets go of the microphone and the audio lane when the server ends the call (set below,
|
|
139
|
+
* once they exist). */
|
|
140
|
+
const releaseRef = useRef<() => void>(() => {});
|
|
116
141
|
const sendAudioRef = useRef<(d: ArrayBuffer) => void>(() => {});
|
|
117
142
|
|
|
118
143
|
const playback = useAudioPlayback();
|
|
@@ -134,6 +159,9 @@ export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
|
|
|
134
159
|
} else if (state === "SESSION_READY") {
|
|
135
160
|
setConnectionStatus("connected");
|
|
136
161
|
} else if (state === "SESSION_ENDED") {
|
|
162
|
+
// THE SERVER ENDED THE CALL (the model hung up, a minute of silence, a timeout): the
|
|
163
|
+
// microphone stops and the lane closes, or the page streams to nobody.
|
|
164
|
+
releaseRef.current();
|
|
137
165
|
setConnectionStatus("ended");
|
|
138
166
|
setAssistantHint(false);
|
|
139
167
|
setUserSpeaking(false);
|
|
@@ -168,55 +196,109 @@ export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
|
|
|
168
196
|
}, [ws.sendAudio]);
|
|
169
197
|
|
|
170
198
|
const capture = useAudioCapture({ onAudioData: handleAudioData });
|
|
199
|
+
useEffect(() => {
|
|
200
|
+
releaseRef.current = () => {
|
|
201
|
+
if (timerRef.current) {
|
|
202
|
+
clearInterval(timerRef.current);
|
|
203
|
+
timerRef.current = null;
|
|
204
|
+
}
|
|
205
|
+
callWorkflowRef.current = null;
|
|
206
|
+
void capture.stopCapture();
|
|
207
|
+
ws.disconnect();
|
|
208
|
+
};
|
|
209
|
+
}, [capture, ws]);
|
|
171
210
|
|
|
172
|
-
const startCall = useCallback(async () => {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
211
|
+
const startCall = useCallback(async (start?: StartCall) => {
|
|
212
|
+
const workflowId = start?.workflowId || session.workflowId;
|
|
213
|
+
const targetTriggerNode = start?.targetTriggerNode || session.targetTriggerNode;
|
|
214
|
+
if (!workflowId) {
|
|
215
|
+
// NOTHING TO CALL: no workflow was named and the session has none. Never a default.
|
|
216
|
+
setError("No workflow to call");
|
|
217
|
+
setConnectionStatus("error");
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
// A CALL ALREADY LIVE IS MOVED, not started again: the line and the microphone stay open,
|
|
221
|
+
// and only the workflow changes. The old voice is cut at once so two are never heard.
|
|
222
|
+
const moving = callWorkflowRef.current !== null;
|
|
223
|
+
if (moving) {
|
|
224
|
+
playback.stopAll();
|
|
225
|
+
setAssistantHint(false);
|
|
226
|
+
setIsLookingUp(false);
|
|
227
|
+
setLookupTool(null);
|
|
228
|
+
} else {
|
|
229
|
+
setConnectionStatus("connecting");
|
|
230
|
+
setError(null);
|
|
231
|
+
setCallDuration(0);
|
|
232
|
+
}
|
|
176
233
|
try {
|
|
177
234
|
// 1. Open the audio lane (sends INIT_SESSION on open) BEFORE the trigger,
|
|
178
235
|
// so the server can route audio the moment the Nova/Grok session is up.
|
|
179
|
-
await ws.connect();
|
|
236
|
+
if (!moving) await ws.connect();
|
|
237
|
+
callWorkflowRef.current = workflowId;
|
|
180
238
|
|
|
181
239
|
// 2. START_CALL via REST — fire-and-forget; the workflow stays open for the call.
|
|
182
240
|
const token = getAccessToken ? await getAccessToken() : null;
|
|
183
241
|
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
184
242
|
if (token) headers["Authorization"] = `Bearer ${token}`;
|
|
185
243
|
const chatId = session.chatId ?? `voice_${session.conversationId}`;
|
|
186
|
-
fetch(`${apiUrl}/api/workflows/${
|
|
244
|
+
fetch(`${apiUrl}/api/workflows/${workflowId}/execute`, {
|
|
187
245
|
method: "POST",
|
|
188
246
|
headers,
|
|
189
247
|
body: JSON.stringify({
|
|
190
248
|
input: {
|
|
191
|
-
message: "Start call",
|
|
249
|
+
message: start?.message ?? "Start call",
|
|
192
250
|
chatId,
|
|
193
251
|
conversationId: session.conversationId,
|
|
194
252
|
userId: session.userId,
|
|
195
253
|
providerId: "gravity-voice",
|
|
196
254
|
isAudio: true,
|
|
197
255
|
metadata: {
|
|
198
|
-
|
|
256
|
+
...(start?.metadata ?? {}),
|
|
257
|
+
targetTriggerNode,
|
|
199
258
|
action: "START_CALL",
|
|
200
259
|
isAction: true,
|
|
201
|
-
workflowId
|
|
260
|
+
workflowId,
|
|
202
261
|
continuousStream: true,
|
|
203
262
|
},
|
|
204
263
|
},
|
|
205
264
|
conversationId: session.conversationId,
|
|
206
265
|
}),
|
|
207
266
|
}).catch((e) => console.error("[voice] START_CALL error:", e));
|
|
267
|
+
if (moving) return;
|
|
208
268
|
|
|
209
269
|
// 3. Start streaming the mic.
|
|
210
270
|
await capture.startCapture();
|
|
211
271
|
timerRef.current = setInterval(() => setCallDuration((p) => p + 1), 1000);
|
|
212
272
|
setConnectionStatus("connected");
|
|
213
273
|
} catch (err: unknown) {
|
|
274
|
+
callWorkflowRef.current = null;
|
|
214
275
|
setError(err instanceof Error ? err.message : "Failed to start call");
|
|
215
276
|
setConnectionStatus("error");
|
|
216
277
|
setIsLookingUp(false);
|
|
217
278
|
setLookupTool(null);
|
|
218
279
|
}
|
|
219
|
-
}, [ws, capture, apiUrl, session, getAccessToken]);
|
|
280
|
+
}, [ws, capture, playback, apiUrl, session, getAccessToken]);
|
|
281
|
+
|
|
282
|
+
// JOIN, NOT START. The workflow is already running and its voice node has opened the vendor
|
|
283
|
+
// session on this conversation; all that is missing is a browser on the audio lane. So open
|
|
284
|
+
// the lane and the microphone, and send no execute: a second one would start a second run.
|
|
285
|
+
const joinCall = useCallback(async (workflowId?: string) => {
|
|
286
|
+
if (callWorkflowRef.current !== null) return;
|
|
287
|
+
callWorkflowRef.current = workflowId ?? session.workflowId ?? "";
|
|
288
|
+
setConnectionStatus("connecting");
|
|
289
|
+
setError(null);
|
|
290
|
+
setCallDuration(0);
|
|
291
|
+
try {
|
|
292
|
+
await ws.connect();
|
|
293
|
+
await capture.startCapture();
|
|
294
|
+
timerRef.current = setInterval(() => setCallDuration((p) => p + 1), 1000);
|
|
295
|
+
setConnectionStatus("connected");
|
|
296
|
+
} catch (err: unknown) {
|
|
297
|
+
callWorkflowRef.current = null;
|
|
298
|
+
setError(err instanceof Error ? err.message : "Failed to join call");
|
|
299
|
+
setConnectionStatus("error");
|
|
300
|
+
}
|
|
301
|
+
}, [ws, capture, session.workflowId]);
|
|
220
302
|
|
|
221
303
|
const endCall = useCallback(async () => {
|
|
222
304
|
if (timerRef.current) {
|
|
@@ -226,7 +308,8 @@ export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
|
|
|
226
308
|
await capture.stopCapture();
|
|
227
309
|
playback.stopAll();
|
|
228
310
|
// END via WS control — NOT a new REST execute (that would start a new workflow).
|
|
229
|
-
ws.sendControl("AUDIO_CONTROL", { command: "stop", workflowId: session.workflowId });
|
|
311
|
+
ws.sendControl("AUDIO_CONTROL", { command: "stop", workflowId: callWorkflowRef.current ?? session.workflowId });
|
|
312
|
+
callWorkflowRef.current = null;
|
|
230
313
|
ws.disconnect();
|
|
231
314
|
setConnectionStatus("ended");
|
|
232
315
|
setIsMuted(false);
|
|
@@ -335,5 +418,5 @@ export function useVoiceService(config: UseVoiceServiceConfig): VoiceService {
|
|
|
335
418
|
store?.mergeAppState(state as unknown as Record<string, unknown>);
|
|
336
419
|
}, [store, state]);
|
|
337
420
|
|
|
338
|
-
return { state, startCall, endCall, toggleMute };
|
|
421
|
+
return { state, startCall, joinCall, endCall, toggleMute, unlock: playback.unlock };
|
|
339
422
|
}
|
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE PAGE'S OWN TEMPLATES (docs/unoverse/design/sdk/UNOVERSE_HOST_PAGE.md rule 3).
|
|
3
|
+
*
|
|
4
|
+
* A website that is the app marks where its unoverse templates go:
|
|
5
|
+
*
|
|
6
|
+
* <div data-unoverse-interface="unoverse/presence-avatar"></div>
|
|
7
|
+
*
|
|
8
|
+
* This runtime is already loaded (the embed started the app with no chrome). It reads each
|
|
9
|
+
* marked template by URI over MCP, exactly as it reads the app, and draws it INTO the page's
|
|
10
|
+
* element, in a shadow DOM (UnoverseComponent's own IsolatedRoot). One runtime, many shadow
|
|
11
|
+
* roots: one client, one conversation, one sign-in for every part of the page.
|
|
12
|
+
*
|
|
13
|
+
* A template ARRIVES WITH ITS PAGE, as Studio previews it and as a delivery lands it: one
|
|
14
|
+
* instance per linked part (`static:`), at its face, with the part's own values
|
|
15
|
+
* (the served `sections` and their `seeds`), then the template itself. Drawn bare, its parts'
|
|
16
|
+
* slots hold nothing and only the frame shows.
|
|
17
|
+
*
|
|
18
|
+
* The frame shares the page's origin (EMBED_CLIENT.md §1), so the page's document is
|
|
19
|
+
* reachable from here; React portals listen for events on their own container, so a part
|
|
20
|
+
* drawn into the page is as live as one drawn in the frame.
|
|
21
|
+
*/
|
|
22
|
+
import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from "react";
|
|
23
|
+
import { createPortal } from "react-dom";
|
|
24
|
+
import { ComponentStore, UnoverseComponent, type ActionHandler, type ActionMeta, type UnoverseClient, type UnoverseDefinition, type VoiceService } from "./lib";
|
|
25
|
+
|
|
26
|
+
const MARK = "data-unoverse-interface";
|
|
27
|
+
const CHAT = "page";
|
|
28
|
+
const TEMPLATE_NODE = "~template";
|
|
29
|
+
|
|
30
|
+
/** The page this runtime serves, or null when there is none it may reach (an MCP host). */
|
|
31
|
+
function pageDocument(): Document | null {
|
|
32
|
+
try {
|
|
33
|
+
return window.parent !== window ? window.parent.document : null;
|
|
34
|
+
} catch {
|
|
35
|
+
return null; // a cross-origin host: there is no page of ours to draw into
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* WHICH TEMPLATE THE ARROW KEYS BELONG TO: exactly one. The one the visitor last pressed in;
|
|
41
|
+
* before any press, the one taking the most of the screen. Every template's state is its
|
|
42
|
+
* own, so a key must never move two (the presentation's slides moved the avatar's steps).
|
|
43
|
+
*/
|
|
44
|
+
let engaged: HTMLElement | null = null;
|
|
45
|
+
function keyOwner(doc: Document): HTMLElement | null {
|
|
46
|
+
const full = doc.fullscreenElement as HTMLElement | null;
|
|
47
|
+
if (full?.hasAttribute(MARK)) return full; // a fullscreen template is the only one on screen
|
|
48
|
+
if (engaged && doc.contains(engaged)) return engaged;
|
|
49
|
+
const vh = doc.defaultView?.innerHeight ?? 0;
|
|
50
|
+
let best: HTMLElement | null = null;
|
|
51
|
+
let area = 0;
|
|
52
|
+
for (const el of doc.querySelectorAll<HTMLElement>(`[${MARK}]`)) {
|
|
53
|
+
const r = el.getBoundingClientRect();
|
|
54
|
+
const seen = Math.max(0, Math.min(r.bottom, vh) - Math.max(r.top, 0)) * r.width;
|
|
55
|
+
if (seen > area) (area = seen), (best = el);
|
|
56
|
+
}
|
|
57
|
+
return best;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** A page template's address, from the name its element is marked with. */
|
|
61
|
+
const uriOf = (name: string) => (name.includes("://") ? name : `unoverse://templates/${name}`);
|
|
62
|
+
|
|
63
|
+
/** THE AVATARS: page templates whose steps are the call's phases (`listening`, `speaking`).
|
|
64
|
+
* They are how the page is spoken to, so they are never the task in view; they are the ones
|
|
65
|
+
* that follow the call. */
|
|
66
|
+
const avatars = new Set<HTMLElement>();
|
|
67
|
+
const CALL_PHASES = ["listening", "speaking"];
|
|
68
|
+
|
|
69
|
+
/** The workflow a template names in its manifest (UNOVERSE_HOST_PAGE.md rule 4). */
|
|
70
|
+
export type Binding = { workflow?: string; trigger?: string };
|
|
71
|
+
|
|
72
|
+
/** THE TASK IN VIEW: the marked template, not an avatar, spanning the middle of the screen. */
|
|
73
|
+
function taskInView(doc: Document): HTMLElement | null {
|
|
74
|
+
const mid = (doc.defaultView?.innerHeight ?? 0) / 2;
|
|
75
|
+
for (const el of doc.querySelectorAll<HTMLElement>(`[${MARK}]`)) {
|
|
76
|
+
if (avatars.has(el)) continue;
|
|
77
|
+
const r = el.getBoundingClientRect();
|
|
78
|
+
if (r.top <= mid && r.bottom >= mid) return el;
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* THE AVATAR'S SPOT WHILE A TEMPLATE IS FULLSCREEN. The browser shows only the fullscreen
|
|
85
|
+
* element, so the avatar's own element (in the site's navigation) is hidden. A fullscreen
|
|
86
|
+
* state's layout carries a `Place`; it draws this element, and the page's avatar draws there
|
|
87
|
+
* until the place goes. It is the same avatar, so it keeps following the call.
|
|
88
|
+
*/
|
|
89
|
+
let dock: HTMLElement | null = null;
|
|
90
|
+
const dockWatchers = new Set<() => void>();
|
|
91
|
+
const setDock = (at: HTMLElement | null) => {
|
|
92
|
+
if (at === dock) return;
|
|
93
|
+
dock = at;
|
|
94
|
+
dockWatchers.forEach((f) => f());
|
|
95
|
+
};
|
|
96
|
+
const watchDock = (f: () => void) => {
|
|
97
|
+
dockWatchers.add(f);
|
|
98
|
+
return () => void dockWatchers.delete(f);
|
|
99
|
+
};
|
|
100
|
+
const drawDock = () => <div ref={setDock} />;
|
|
101
|
+
|
|
102
|
+
type Section = { kind?: string; face?: string; ref?: string; preview?: string[]; seeds?: Record<string, Record<string, unknown>> };
|
|
103
|
+
|
|
104
|
+
/** One marked template: read it, give it its parts, draw it. */
|
|
105
|
+
function PageTemplate({ client, uri, el, voice, waiting, onBinding, onCall, appStore }: { client: UnoverseClient; uri: string; el: HTMLElement; voice: VoiceService; waiting: boolean; onBinding: (uri: string, binding: Binding) => void; onCall: ActionHandler; appStore: ComponentStore }) {
|
|
106
|
+
const store = useMemo(() => new ComponentStore(), [uri]);
|
|
107
|
+
const [ready, setReady] = useState(false);
|
|
108
|
+
const partsRef = useRef<string[]>([]);
|
|
109
|
+
const defRef = useRef<{ stateTree?: { name: string; on?: string; initial?: string; states?: { name: string }[]; fullscreen?: boolean }[] } | null>(null);
|
|
110
|
+
|
|
111
|
+
useEffect(() => {
|
|
112
|
+
let alive = true;
|
|
113
|
+
client
|
|
114
|
+
.readDefinition(uri)
|
|
115
|
+
.then((def) => {
|
|
116
|
+
if (!alive || !def) return;
|
|
117
|
+
const d = def as UnoverseDefinition & { defaultLayout?: string; layouts?: Record<string, unknown>; sections?: Record<string, Section[]>; binding?: Binding };
|
|
118
|
+
if (d.binding?.workflow) onBinding(uri, d.binding);
|
|
119
|
+
defRef.current = d as typeof defRef.current;
|
|
120
|
+
const base = d.defaultLayout ?? (d.layouts ? Object.keys(d.layouts)[0] : undefined);
|
|
121
|
+
partsRef.current = [];
|
|
122
|
+
for (const sec of (base && d.sections?.[base]) || []) {
|
|
123
|
+
const parts = sec.ref ? [sec.ref] : sec.kind === "director" ? sec.preview ?? [] : [];
|
|
124
|
+
parts.forEach((ref, i) => {
|
|
125
|
+
const nodeId = `${sec.face}~${ref}~${i}`;
|
|
126
|
+
partsRef.current.push(nodeId);
|
|
127
|
+
store.apply({ type: "COMPONENT_INIT", chatId: CHAT, nodeId, component: { type: ref, props: sec.seeds?.[ref] ?? {} } });
|
|
128
|
+
if (sec.face) store.mergeComponentState(CHAT, nodeId, { state: sec.face });
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
const ref = uri.replace(/^unoverse:\/\/templates\//, "");
|
|
132
|
+
store.apply({ type: "COMPONENT_INIT", chatId: CHAT, nodeId: TEMPLATE_NODE, component: { type: ref, componentUrl: uri, props: {} } });
|
|
133
|
+
if (base) store.mergeComponentState(CHAT, TEMPLATE_NODE, { state: base });
|
|
134
|
+
if ((d.stateTree ?? []).some((t) => CALL_PHASES.every((p) => t.states?.some((x) => x.name === p)))) avatars.add(el);
|
|
135
|
+
setReady(true);
|
|
136
|
+
})
|
|
137
|
+
.catch((e) => console.warn(`[unoverse] page template "${uri}" failed to load:`, e));
|
|
138
|
+
return () => {
|
|
139
|
+
alive = false;
|
|
140
|
+
};
|
|
141
|
+
}, [client, uri, store]);
|
|
142
|
+
|
|
143
|
+
// A TEMPLATE FOLLOWS ITS TASK. Whatever runs the task publishes where it is as
|
|
144
|
+
// `tasks.<its URI>` in the app state (SIGNAL.md 6c); this template moves to that step, the
|
|
145
|
+
// same write its own Next makes, whether the move came from a voice or a chat.
|
|
146
|
+
// Its `props` too, keyed by part: `{ <part>: values }` merges into that part (a template's
|
|
147
|
+
// content lives in its parts), once per change, so a template already on the page is
|
|
148
|
+
// hydrated in place.
|
|
149
|
+
//
|
|
150
|
+
// THE SAME TEMPLATE, STREAMED. When a run renders this template, it streams into the
|
|
151
|
+
// client's store like any interface. It is already on the page, so it is not drawn again:
|
|
152
|
+
// this copy hydrates from it, each part's values onto that part. A copy still on its first
|
|
153
|
+
// step (the wait, such as `finding`) moves to the next step as its values land; nothing
|
|
154
|
+
// outside the template moves it.
|
|
155
|
+
useEffect(() => {
|
|
156
|
+
if (!ready) return;
|
|
157
|
+
let lastProps = "";
|
|
158
|
+
let lastStreamed = "";
|
|
159
|
+
const fillParts = (parts: Record<string, unknown>) => {
|
|
160
|
+
for (const [part, values] of Object.entries(parts)) {
|
|
161
|
+
if (!values || typeof values !== "object" || Array.isArray(values)) continue;
|
|
162
|
+
for (const nodeId of partsRef.current.filter((id) => id.startsWith(`${part}~`))) store.mergeComponentState(CHAT, nodeId, values as Record<string, unknown>);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
const moveTo = (at: unknown) => {
|
|
166
|
+
if (typeof at !== "string") return;
|
|
167
|
+
const tree = defRef.current?.stateTree?.find((t) => t.on && t.states?.some((x) => x.name === at));
|
|
168
|
+
if (!tree?.on) return;
|
|
169
|
+
const tpl = store.get(CHAT, TEMPLATE_NODE) as Record<string, unknown> | undefined;
|
|
170
|
+
if (tpl?.[tree.on] !== at) store.mergeComponentState(CHAT, TEMPLATE_NODE, { [tree.on]: at });
|
|
171
|
+
};
|
|
172
|
+
const follow = () => {
|
|
173
|
+
const task = ((appStore.getAppState().tasks ?? {}) as Record<string, { step?: string; props?: Record<string, unknown> }>)[uri];
|
|
174
|
+
if (task?.props && typeof task.props === "object") {
|
|
175
|
+
const next = JSON.stringify(task.props);
|
|
176
|
+
if (next !== lastProps) {
|
|
177
|
+
lastProps = next;
|
|
178
|
+
fillParts(task.props);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
moveTo(task?.step);
|
|
182
|
+
for (const r of appStore.getResponses()) {
|
|
183
|
+
for (const pointer of r.components) {
|
|
184
|
+
const { chatId, nodeId } = appStore.split(pointer);
|
|
185
|
+
if ((appStore.getComponentUrl(chatId, nodeId) ?? "").split("#")[0] !== uri) continue;
|
|
186
|
+
const streamed = appStore.get(chatId, nodeId) ?? {};
|
|
187
|
+
const next = JSON.stringify(streamed);
|
|
188
|
+
if (next === lastStreamed) continue;
|
|
189
|
+
lastStreamed = next;
|
|
190
|
+
if (!streamed.state || typeof streamed.state !== "object") continue;
|
|
191
|
+
fillParts(streamed.state as Record<string, unknown>);
|
|
192
|
+
const tree = defRef.current?.stateTree?.find((t) => t.on && (t.states?.length ?? 0) > 1);
|
|
193
|
+
const tpl = store.get(CHAT, TEMPLATE_NODE) as Record<string, unknown> | undefined;
|
|
194
|
+
if (tree?.on && (tpl?.[tree.on] ?? tree.initial) === tree.initial) moveTo(tree.states![1].name);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
follow();
|
|
199
|
+
return appStore.subscribe(follow);
|
|
200
|
+
}, [ready, appStore, store, uri]);
|
|
201
|
+
|
|
202
|
+
// THE ARROW KEYS STEP IT, on the PAGE. Left and Right move a template's steps (the served
|
|
203
|
+
// step tree: `on` + its substates), the same write its own Back and Next make. The keys are
|
|
204
|
+
// the page's, so they are heard on the page's document, not this frame's. Typing in a field
|
|
205
|
+
// never steps, and only the template that owns the keys moves (keyOwner).
|
|
206
|
+
useEffect(() => {
|
|
207
|
+
const doc = pageDocument();
|
|
208
|
+
if (!doc || !ready) return;
|
|
209
|
+
const onKey = (e: KeyboardEvent) => {
|
|
210
|
+
if (e.key !== "ArrowRight" && e.key !== "ArrowLeft") return;
|
|
211
|
+
if (keyOwner(doc) !== el) return;
|
|
212
|
+
const t = e.target as HTMLElement | null;
|
|
213
|
+
if (t && (t.isContentEditable || /^(INPUT|TEXTAREA|SELECT)$/.test(t.tagName))) return;
|
|
214
|
+
const tpl = store.get(CHAT, TEMPLATE_NODE) as Record<string, unknown> | undefined;
|
|
215
|
+
const def = defRef.current;
|
|
216
|
+
const face = String(tpl?.state ?? "");
|
|
217
|
+
const tree = def?.stateTree?.find((s) => s.name === face);
|
|
218
|
+
const steps = tree?.states?.map((s) => s.name) ?? [];
|
|
219
|
+
if (!tree?.on || steps.length < 2) return;
|
|
220
|
+
const cur = String(tpl?.[tree.on] ?? tree.initial ?? steps[0]);
|
|
221
|
+
const i = steps.indexOf(cur) + (e.key === "ArrowRight" ? 1 : -1);
|
|
222
|
+
if (i < 0 || i >= steps.length) return;
|
|
223
|
+
e.preventDefault();
|
|
224
|
+
store.mergeComponentState(CHAT, TEMPLATE_NODE, { [tree.on]: steps[i] });
|
|
225
|
+
};
|
|
226
|
+
doc.addEventListener("keydown", onKey);
|
|
227
|
+
return () => doc.removeEventListener("keydown", onKey);
|
|
228
|
+
}, [ready, store, el]);
|
|
229
|
+
|
|
230
|
+
// FULLSCREEN IS A STATE. Entering a state marked `fullscreen` takes this template's element
|
|
231
|
+
// fullscreen; leaving it hands the screen back. The request is made inside the store write
|
|
232
|
+
// the visitor's click made (setValue notifies synchronously), because a browser grants
|
|
233
|
+
// fullscreen only to a click. Refused, or left with Esc, the template goes back to the
|
|
234
|
+
// state it came from. The step is one value across states, so the slide stays put.
|
|
235
|
+
useEffect(() => {
|
|
236
|
+
const doc = pageDocument();
|
|
237
|
+
if (!doc || !ready) return;
|
|
238
|
+
const states = () => defRef.current?.stateTree ?? [];
|
|
239
|
+
const isFull = (name: string) => states().some((t) => t.name === name && t.fullscreen);
|
|
240
|
+
const face = () => String((store.get(CHAT, TEMPLATE_NODE) as Record<string, unknown> | undefined)?.state ?? "");
|
|
241
|
+
let from = isFull(face()) ? (states().find((t) => !t.fullscreen)?.name ?? "") : face();
|
|
242
|
+
let asking = false;
|
|
243
|
+
const back = () => store.mergeComponentState(CHAT, TEMPLATE_NODE, { state: from });
|
|
244
|
+
const sync = () => {
|
|
245
|
+
const now = face();
|
|
246
|
+
if (!isFull(now)) {
|
|
247
|
+
from = now;
|
|
248
|
+
if (doc.fullscreenElement === el) void doc.exitFullscreen();
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (doc.fullscreenElement === el || asking) return;
|
|
252
|
+
asking = true;
|
|
253
|
+
el.requestFullscreen()
|
|
254
|
+
.catch(back)
|
|
255
|
+
.finally(() => (asking = false));
|
|
256
|
+
};
|
|
257
|
+
const onChange = () => {
|
|
258
|
+
if (doc.fullscreenElement !== el && !asking && isFull(face())) back();
|
|
259
|
+
};
|
|
260
|
+
const stop = store.subscribe(sync);
|
|
261
|
+
doc.addEventListener("fullscreenchange", onChange);
|
|
262
|
+
return () => {
|
|
263
|
+
stop();
|
|
264
|
+
doc.removeEventListener("fullscreenchange", onChange);
|
|
265
|
+
};
|
|
266
|
+
}, [ready, store, el]);
|
|
267
|
+
|
|
268
|
+
// WHILE A TEMPLATE IS FULLSCREEN, AN AVATAR DRAWS IN ITS PLACE (drawDock).
|
|
269
|
+
const docked = useSyncExternalStore(watchDock, () => dock);
|
|
270
|
+
|
|
271
|
+
// AN AVATAR FOLLOWS THE CALL: its steps are the call's phases, so its step is the call's
|
|
272
|
+
// state. `waiting` rides beside it, on any step: a task in view that the call is not working
|
|
273
|
+
// yet. The avatar draws it as a pulse, and a press on the pulse starts or moves the call.
|
|
274
|
+
const callState = voice.state.callState;
|
|
275
|
+
useEffect(() => {
|
|
276
|
+
if (!ready || !avatars.has(el)) return;
|
|
277
|
+
const tree = defRef.current?.stateTree?.find((t) => CALL_PHASES.every((p) => t.states?.some((x) => x.name === p)));
|
|
278
|
+
if (!tree?.on) return;
|
|
279
|
+
store.mergeComponentState(CHAT, TEMPLATE_NODE, { [tree.on]: callState, waiting });
|
|
280
|
+
}, [ready, store, el, callState, waiting]);
|
|
281
|
+
|
|
282
|
+
if (!ready) return null;
|
|
283
|
+
const drawn = <UnoverseComponent client={client} store={store} chatId={CHAT} nodeId={TEMPLATE_NODE} onAction={onCall} place={drawDock} appStore={appStore} />;
|
|
284
|
+
return docked && avatars.has(el) ? createPortal(drawn, docked) : drawn;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* ONE APP STATE FOR THE WHOLE PAGE. Each template keeps its own slice in its own store, and
|
|
289
|
+
* all of them read and write the app's state in `appStore`, the store the call and the
|
|
290
|
+
* workflow write to. A name one template saves is a name every other template can show.
|
|
291
|
+
*/
|
|
292
|
+
export function PageInterfaces({ org, client, voice, appStore, toServer, startTask }: { org: string; client: UnoverseClient; voice: VoiceService; appStore: ComponentStore; toServer: (t: string, data: Record<string, unknown>, meta?: ActionMeta) => void; startTask: (binding: Binding, task: string) => void }) {
|
|
293
|
+
const [targets, setTargets] = useState<HTMLElement[]>([]);
|
|
294
|
+
// The task in view, by its URI, measured again as the page scrolls or resizes.
|
|
295
|
+
const [inView, setInView] = useState<string | null>(null);
|
|
296
|
+
// Each template's own workflow, as it loads; a template without one is never called.
|
|
297
|
+
const [bindings, setBindings] = useState<Record<string, Binding>>({});
|
|
298
|
+
const onBinding = useCallback((uri: string, b: Binding) => setBindings((prev) => (prev[uri] ? prev : { ...prev, [uri]: b })), []);
|
|
299
|
+
// The tasks this page has started. Each starts once: a second press starts nothing. The ref
|
|
300
|
+
// holds it at once, so two quick presses are still one; the state redraws the avatar.
|
|
301
|
+
const started = useRef(new Set<string>());
|
|
302
|
+
const [, setStarted] = useState(0);
|
|
303
|
+
|
|
304
|
+
// THE AVATAR STARTS THE TASK'S WORKFLOW (UNOVERSE_HOST_PAGE.md rule 4), like any button. The
|
|
305
|
+
// page connects to nothing until it is pressed. A task in view with a workflow, that is not
|
|
306
|
+
// already running, is WAITING: a press starts its workflow at the template's trigger. A voice
|
|
307
|
+
// in that workflow is the workflow's own. Nothing in view, or nothing bound: nothing.
|
|
308
|
+
const target = inView && bindings[inView]?.workflow ? inView : null;
|
|
309
|
+
const waiting = !!target && !started.current.has(target);
|
|
310
|
+
// A TEMPLATE'S OWN ACTIONS go where the app's go: its `submit` answers the task that is waiting
|
|
311
|
+
// on it, exactly as in the app (appEngine `toServer`). The call's three are the page's.
|
|
312
|
+
const onCall: ActionHandler = (spec, data, meta) => {
|
|
313
|
+
const action = typeof spec === "string" ? spec : spec?.type;
|
|
314
|
+
if (action === "submit" || action === "cancel") toServer(action, data, meta);
|
|
315
|
+
else if (action === "startCall") {
|
|
316
|
+
if (!target || started.current.has(target)) return;
|
|
317
|
+
started.current.add(target);
|
|
318
|
+
setStarted((n) => n + 1);
|
|
319
|
+
// The press is the click the browser needs before it plays a voice the workflow may have.
|
|
320
|
+
voice.unlock();
|
|
321
|
+
startTask(bindings[target], target);
|
|
322
|
+
} else if (action === "endCall") {
|
|
323
|
+
void voice.endCall();
|
|
324
|
+
} else if (action === "toggleMute") voice.toggleMute();
|
|
325
|
+
// Another app on the page, opened or closed by its `<org>/<app>` (its own tag, EMBED_CLIENT.md).
|
|
326
|
+
else if (action === "toggleApp") {
|
|
327
|
+
const app = (data as { app?: unknown } | undefined)?.app;
|
|
328
|
+
if (typeof app === "string") (pageDocument()?.defaultView as (Window & { unoverseApps?: Record<string, { toggle(): void }> }) | undefined)?.unoverseApps?.[app]?.toggle();
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
|
|
332
|
+
useEffect(() => {
|
|
333
|
+
const doc = pageDocument();
|
|
334
|
+
const win = doc?.defaultView;
|
|
335
|
+
if (!doc || !win) return;
|
|
336
|
+
const measure = () => {
|
|
337
|
+
const name = taskInView(doc)?.getAttribute(MARK);
|
|
338
|
+
setInView(name ? uriOf(name) : null);
|
|
339
|
+
};
|
|
340
|
+
measure();
|
|
341
|
+
win.addEventListener("scroll", measure, { passive: true });
|
|
342
|
+
win.addEventListener("resize", measure);
|
|
343
|
+
return () => {
|
|
344
|
+
win.removeEventListener("scroll", measure);
|
|
345
|
+
win.removeEventListener("resize", measure);
|
|
346
|
+
};
|
|
347
|
+
}, [targets]);
|
|
348
|
+
|
|
349
|
+
useEffect(() => {
|
|
350
|
+
const doc = pageDocument();
|
|
351
|
+
if (!doc) return;
|
|
352
|
+
const scan = () => {
|
|
353
|
+
// Only this app's own org's templates: another app on the page draws its own.
|
|
354
|
+
const found = [...doc.querySelectorAll<HTMLElement>(`[${MARK}]`)].filter((el) => el.getAttribute(MARK)?.startsWith(`${org}/`));
|
|
355
|
+
setTargets((prev) => (prev.length === found.length && prev.every((el, i) => el === found[i]) ? prev : found));
|
|
356
|
+
};
|
|
357
|
+
scan();
|
|
358
|
+
// The page may render its parts after this runtime starts (a client-rendered page),
|
|
359
|
+
// so it keeps looking; only a change in the marked set re-renders.
|
|
360
|
+
const watch = new MutationObserver(scan);
|
|
361
|
+
watch.observe(doc.body, { childList: true, subtree: true });
|
|
362
|
+
// A press anywhere inside a marked template makes it the one the keys move; a press
|
|
363
|
+
// outside every template hands the keys back to the one most in view.
|
|
364
|
+
const onPress = (e: Event) => {
|
|
365
|
+
engaged = (e.composedPath().find((n) => n instanceof HTMLElement && n.hasAttribute(MARK)) as HTMLElement | undefined) ?? null;
|
|
366
|
+
};
|
|
367
|
+
doc.addEventListener("pointerdown", onPress, true);
|
|
368
|
+
return () => {
|
|
369
|
+
watch.disconnect();
|
|
370
|
+
doc.removeEventListener("pointerdown", onPress, true);
|
|
371
|
+
};
|
|
372
|
+
}, []);
|
|
373
|
+
|
|
374
|
+
return (
|
|
375
|
+
<>
|
|
376
|
+
{targets.map((el) => {
|
|
377
|
+
const name = el.getAttribute(MARK) ?? "";
|
|
378
|
+
if (!name) return null;
|
|
379
|
+
return createPortal(<PageTemplate client={client} uri={uriOf(name)} el={el} voice={voice} waiting={waiting} onBinding={onBinding} onCall={onCall} appStore={appStore} />, el, name);
|
|
380
|
+
})}
|
|
381
|
+
</>
|
|
382
|
+
);
|
|
383
|
+
}
|