@unoverse-platform/studio 0.1.103 → 0.1.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local/sample-app/chat/manifest.yaml +2 -2
- package/local/sample-component/text-chat/components/suggestions.yaml +1 -1
- package/local/sample-component/text-chat/layouts/chat-welcome.yaml +1 -1
- package/local/sample-component/text-chat/text-chat.yaml +3 -0
- package/local/scaffold.d.mts +5 -2
- package/local/scaffold.mjs +68 -1
- package/package.json +4 -2
- package/screens/AtomsView.tsx +2 -1
- package/screens/ComponentsView.tsx +2 -1
- package/screens/NativeAppHost.tsx +1 -2
- package/screens/TemplatesLibraryView.tsx +14 -1
- package/screens/TemplatesView.tsx +2 -1
- package/screens/index.ts +4 -0
- package/screens/react/unoverse.tsx +281 -0
- package/vendor/base/src/agent-mcp/index.ts +1 -1
- package/vendor/base/src/agent-mcp/invoke.ts +6 -2
- package/vendor/base/src/agent-mcp/tools.ts +14 -5
- package/vendor/base/src/agent-mcp/types.ts +6 -0
- package/vendor/base/src/auth/auth.ts +90 -23
- package/vendor/base/src/definitions/load/compile/sections.ts +79 -0
- package/vendor/base/src/definitions/load/compile/template.ts +23 -2
- package/vendor/base/src/definitions/model.ts +9 -0
- package/vendor/base/src/definitions/registry/catalog.ts +6 -7
- package/vendor/base/src/executor.ts +25 -1
- package/vendor/base/src/lint/design/index.mjs +9 -0
- package/vendor/base/src/manifests/executor/toolBridge.ts +25 -19
- package/vendor/base/src/manifests/runtime/channels/final.ts +4 -1
- package/vendor/base/src/manifests/runtime/duplex/session.ts +3 -9
- package/vendor/base/src/manifests/runtime/tools/toolloop.ts +28 -25
- package/vendor/base/src/platform/serviceCalls.ts +13 -0
- package/vendor/sdk/appEngine.ts +363 -0
- package/vendor/sdk/lib/actions.ts +36 -0
- package/vendor/sdk/lib/citations.ts +48 -0
- package/vendor/sdk/lib/core/apps.ts +18 -0
- package/vendor/sdk/lib/core/client.ts +3 -3
- package/vendor/sdk/lib/core/store.ts +6 -0
- package/vendor/sdk/lib/dom/app.ts +546 -0
- package/vendor/sdk/lib/dom/chrome.ts +125 -0
- package/vendor/sdk/lib/dom/component.ts +333 -0
- package/vendor/sdk/lib/dom/connection.ts +309 -0
- package/vendor/sdk/lib/dom/element.ts +69 -0
- package/vendor/sdk/lib/dom/isolate.ts +37 -0
- package/vendor/sdk/lib/dom/markdown.ts +388 -0
- package/vendor/sdk/lib/dom/markdownParse.ts +440 -0
- package/vendor/sdk/lib/dom/mounted.ts +29 -0
- package/vendor/sdk/lib/dom/primitives.ts +689 -0
- package/vendor/sdk/lib/dom/region.ts +195 -0
- package/vendor/sdk/lib/dom/render.ts +177 -0
- package/vendor/sdk/lib/dom/theme.ts +38 -0
- package/vendor/sdk/lib/dom/types.ts +36 -0
- package/vendor/sdk/lib/index.ts +26 -17
- package/vendor/sdk/lib/orb/canvas.ts +139 -0
- package/vendor/sdk/lib/orb/index.ts +8 -18
- package/vendor/sdk/lib/orb/presets.ts +1 -1
- package/vendor/sdk/lib/orb/{types.ts → states.ts} +0 -24
- package/vendor/sdk/lib/realtime/capture.ts +152 -0
- package/vendor/sdk/lib/realtime/playback.ts +129 -0
- package/vendor/sdk/lib/realtime/socket.ts +117 -0
- package/vendor/sdk/lib/style.ts +45 -37
- package/vendor/sdk/lib/voiceService.ts +344 -0
- package/vendor/sdk/main.ts +107 -0
- package/vendor/sdk/mcpAppBridge.ts +125 -102
- package/vendor/sdk/pageInterfaces.ts +452 -0
- package/vendor/sdk/responseView.ts +193 -0
- package/vite.config.ts +3 -3
- package/vendor/sdk/appEngine.tsx +0 -395
- package/vendor/sdk/lib/UnoverseComponent.tsx +0 -325
- package/vendor/sdk/lib/app.tsx +0 -730
- package/vendor/sdk/lib/connection.tsx +0 -433
- package/vendor/sdk/lib/isolate.tsx +0 -70
- package/vendor/sdk/lib/orb/ThinkingOrb.tsx +0 -131
- package/vendor/sdk/lib/orb/theme.ts +0 -83
- package/vendor/sdk/lib/primitives.tsx +0 -943
- package/vendor/sdk/lib/realtime/useAudioCapture.ts +0 -237
- package/vendor/sdk/lib/realtime/useAudioPlayback.ts +0 -157
- package/vendor/sdk/lib/realtime/useRealtimeWebSocket.ts +0 -164
- package/vendor/sdk/lib/render.tsx +0 -244
- package/vendor/sdk/lib/streamed.tsx +0 -30
- package/vendor/sdk/lib/theme.ts +0 -55
- package/vendor/sdk/lib/transition/MountArrangement.tsx +0 -71
- package/vendor/sdk/lib/transition/index.ts +0 -9
- package/vendor/sdk/lib/transition/viewTransition.ts +0 -29
- package/vendor/sdk/lib/voice.tsx +0 -422
- package/vendor/sdk/main.tsx +0 -125
- package/vendor/sdk/pageInterfaces.tsx +0 -388
- package/vendor/sdk/responseView.tsx +0 -82
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import type { ModeOpts } from './engine/profiles';
|
|
7
7
|
import { BASE_PROFILES, scaleCounts, scaleRadii } from './engine/profiles';
|
|
8
8
|
import { RING_PROFILES } from './engine/rings';
|
|
9
|
-
import type { OrbSize, OrbState } from './
|
|
9
|
+
import type { OrbSize, OrbState } from './states';
|
|
10
10
|
|
|
11
11
|
export type ModeKey =
|
|
12
12
|
| 'orbits'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { CSSProperties, CanvasHTMLAttributes } from 'react';
|
|
2
1
|
|
|
3
2
|
/**
|
|
4
3
|
* The nine shipped states — each a hand-tuned animation:
|
|
@@ -59,26 +58,3 @@ export type OrbSize = 64 | 20;
|
|
|
59
58
|
* backgrounds); light renders dark ink (for light backgrounds).
|
|
60
59
|
*/
|
|
61
60
|
export type OrbTheme = 'auto' | 'dark' | 'light';
|
|
62
|
-
|
|
63
|
-
/** Props for the ThinkingOrb React component. */
|
|
64
|
-
export interface ThinkingOrbProps extends Omit<CanvasHTMLAttributes<HTMLCanvasElement>, 'style'> {
|
|
65
|
-
/** Which animation to show. @default 'working' */
|
|
66
|
-
state?: OrbState;
|
|
67
|
-
|
|
68
|
-
/** Tuned size preset — 64 or 20 CSS px. @default 64 */
|
|
69
|
-
size?: OrbSize;
|
|
70
|
-
|
|
71
|
-
/** Theme mode; `auto` detects from the host project. @default 'auto' */
|
|
72
|
-
theme?: OrbTheme;
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Animation speed multiplier on top of the preset's baked speed.
|
|
76
|
-
* @default 1
|
|
77
|
-
*/
|
|
78
|
-
speed?: number;
|
|
79
|
-
|
|
80
|
-
/** Freeze the animation on the current frame. @default false */
|
|
81
|
-
paused?: boolean;
|
|
82
|
-
|
|
83
|
-
style?: CSSProperties;
|
|
84
|
-
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Continuous microphone capture for full-duplex voice. Streams raw mic audio while a call is
|
|
3
|
+
* active; the remote provider (Grok / Nova) does VAD server-side, so nothing is gated here.
|
|
4
|
+
* Audio is downsampled to 16 kHz mono Int16 PCM and emitted in ~80 ms chunks.
|
|
5
|
+
*/
|
|
6
|
+
import { float32ToInt16, downsampleFloat32 } from "./audioUtils";
|
|
7
|
+
|
|
8
|
+
const TARGET_SAMPLE_RATE = 16000;
|
|
9
|
+
// ~80 ms of audio at 16 kHz: 1280 samples
|
|
10
|
+
const CHUNK_SAMPLES = 1280;
|
|
11
|
+
|
|
12
|
+
export class AudioCapture {
|
|
13
|
+
isCapturing = false;
|
|
14
|
+
/** Server-driven (Grok / Nova VAD); the service learns speech from the audio-state events. */
|
|
15
|
+
readonly isSpeaking = false;
|
|
16
|
+
error: string | null = null;
|
|
17
|
+
private stream: MediaStream | null = null;
|
|
18
|
+
private context: AudioContext | null = null;
|
|
19
|
+
private source: MediaStreamAudioSourceNode | null = null;
|
|
20
|
+
private worklet: AudioWorkletNode | null = null;
|
|
21
|
+
private script: ScriptProcessorNode | null = null;
|
|
22
|
+
private pending: number[] = [];
|
|
23
|
+
private muted = false;
|
|
24
|
+
|
|
25
|
+
/** `onChange` hears each change of `isCapturing` or `error`. */
|
|
26
|
+
constructor(private readonly onAudioData: (audioData: ArrayBuffer) => void, private readonly onChange: () => void = () => {}) {}
|
|
27
|
+
|
|
28
|
+
/** Mute or unmute outgoing mic frames (the stream stays open). */
|
|
29
|
+
setMuted(muted: boolean): void {
|
|
30
|
+
this.muted = muted;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// A Float32 frame at the source rate → 16 kHz Int16 chunks, flushed per CHUNK_SAMPLES.
|
|
34
|
+
private frame(frame: Float32Array, fromRate: number): void {
|
|
35
|
+
if (this.muted) return;
|
|
36
|
+
const downsampled = fromRate === TARGET_SAMPLE_RATE ? frame : downsampleFloat32(frame, fromRate, TARGET_SAMPLE_RATE);
|
|
37
|
+
const pending = this.pending;
|
|
38
|
+
for (let i = 0; i < downsampled.length; i++) pending.push(downsampled[i]);
|
|
39
|
+
while (pending.length >= CHUNK_SAMPLES) {
|
|
40
|
+
const int16 = float32ToInt16(new Float32Array(pending.splice(0, CHUNK_SAMPLES)));
|
|
41
|
+
this.onAudioData(int16.buffer.slice(int16.byteOffset, int16.byteOffset + int16.byteLength) as ArrayBuffer);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private async teardown(): Promise<void> {
|
|
46
|
+
try {
|
|
47
|
+
if (this.worklet) {
|
|
48
|
+
this.worklet.port.onmessage = null;
|
|
49
|
+
this.worklet.disconnect();
|
|
50
|
+
this.worklet = null;
|
|
51
|
+
}
|
|
52
|
+
if (this.script) {
|
|
53
|
+
this.script.onaudioprocess = null;
|
|
54
|
+
this.script.disconnect();
|
|
55
|
+
this.script = null;
|
|
56
|
+
}
|
|
57
|
+
if (this.source) {
|
|
58
|
+
this.source.disconnect();
|
|
59
|
+
this.source = null;
|
|
60
|
+
}
|
|
61
|
+
if (this.stream) {
|
|
62
|
+
this.stream.getTracks().forEach((t) => t.stop());
|
|
63
|
+
this.stream = null;
|
|
64
|
+
}
|
|
65
|
+
if (this.context && this.context.state !== "closed") await this.context.close();
|
|
66
|
+
this.context = null;
|
|
67
|
+
this.pending = [];
|
|
68
|
+
} catch (err) {
|
|
69
|
+
console.warn("[AudioCapture] teardown error:", err);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async start(): Promise<{ success: boolean; reason?: string }> {
|
|
74
|
+
if (this.isCapturing || this.stream) return { success: true };
|
|
75
|
+
this.error = null;
|
|
76
|
+
this.onChange();
|
|
77
|
+
try {
|
|
78
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
79
|
+
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
|
80
|
+
});
|
|
81
|
+
this.stream = stream;
|
|
82
|
+
const AudioContextClass =
|
|
83
|
+
(window as unknown as { AudioContext?: typeof AudioContext; webkitAudioContext?: typeof AudioContext }).AudioContext ||
|
|
84
|
+
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
|
85
|
+
const audioContext: AudioContext = new (AudioContextClass as typeof AudioContext)();
|
|
86
|
+
this.context = audioContext;
|
|
87
|
+
const fromRate = audioContext.sampleRate;
|
|
88
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
89
|
+
this.source = source;
|
|
90
|
+
// Prefer AudioWorklet (modern). Fall back to ScriptProcessor.
|
|
91
|
+
let usedWorklet = false;
|
|
92
|
+
if (audioContext.audioWorklet) {
|
|
93
|
+
try {
|
|
94
|
+
const workletCode = `
|
|
95
|
+
class CaptureProcessor extends AudioWorkletProcessor {
|
|
96
|
+
process(inputs) {
|
|
97
|
+
const ch = inputs[0] && inputs[0][0];
|
|
98
|
+
if (ch && ch.length) {
|
|
99
|
+
this.port.postMessage(new Float32Array(ch));
|
|
100
|
+
}
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
registerProcessor('gravity-capture-processor', CaptureProcessor);
|
|
105
|
+
`;
|
|
106
|
+
const url = URL.createObjectURL(new Blob([workletCode], { type: "application/javascript" }));
|
|
107
|
+
await audioContext.audioWorklet.addModule(url);
|
|
108
|
+
URL.revokeObjectURL(url);
|
|
109
|
+
const node = new AudioWorkletNode(audioContext, "gravity-capture-processor");
|
|
110
|
+
node.port.onmessage = (ev: MessageEvent<Float32Array>) => this.frame(ev.data, fromRate);
|
|
111
|
+
source.connect(node);
|
|
112
|
+
// The worklet must connect to destination to keep ticking: through a muted gain,
|
|
113
|
+
// so the mic is not played back.
|
|
114
|
+
const sink = audioContext.createGain();
|
|
115
|
+
sink.gain.value = 0;
|
|
116
|
+
node.connect(sink).connect(audioContext.destination);
|
|
117
|
+
this.worklet = node;
|
|
118
|
+
usedWorklet = true;
|
|
119
|
+
} catch (err) {
|
|
120
|
+
console.warn("[AudioCapture] AudioWorklet unavailable, falling back to ScriptProcessor:", err);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (!usedWorklet) {
|
|
124
|
+
const node = audioContext.createScriptProcessor(4096, 1, 1);
|
|
125
|
+
node.onaudioprocess = (ev) => this.frame(new Float32Array(ev.inputBuffer.getChannelData(0)), fromRate);
|
|
126
|
+
source.connect(node);
|
|
127
|
+
const sink = audioContext.createGain();
|
|
128
|
+
sink.gain.value = 0;
|
|
129
|
+
node.connect(sink).connect(audioContext.destination);
|
|
130
|
+
this.script = node;
|
|
131
|
+
}
|
|
132
|
+
this.isCapturing = true;
|
|
133
|
+
this.onChange();
|
|
134
|
+
return { success: true };
|
|
135
|
+
} catch (err: unknown) {
|
|
136
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
137
|
+
console.error("[AudioCapture] Failed to start:", err);
|
|
138
|
+
this.error = msg;
|
|
139
|
+
this.onChange();
|
|
140
|
+
await this.teardown();
|
|
141
|
+
return { success: false, reason: msg };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async stop(): Promise<{ success: boolean; reason?: string }> {
|
|
146
|
+
if (!this.stream && !this.context) return { success: false, reason: "not_running" };
|
|
147
|
+
await this.teardown();
|
|
148
|
+
this.isCapturing = false;
|
|
149
|
+
this.onChange();
|
|
150
|
+
return { success: true };
|
|
151
|
+
}
|
|
152
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LPCM audio playback with timeline-based scheduling (Web Audio API, low latency). Audio from
|
|
3
|
+
* the server (Nova / Grok): LPCM Int16, 24 kHz, mono.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** How far ahead the first frame is scheduled when nothing is queued. */
|
|
7
|
+
const START_LEAD_SECONDS = 0.2;
|
|
8
|
+
|
|
9
|
+
/** Decode LPCM (Int16, 24 kHz, mono) → AudioBuffer. */
|
|
10
|
+
function decodeLPCM(audioContext: AudioContext, data: ArrayBuffer): AudioBuffer {
|
|
11
|
+
const int16 = new Int16Array(data);
|
|
12
|
+
const audioBuffer = audioContext.createBuffer(1, int16.length, 24000);
|
|
13
|
+
const channelData = audioBuffer.getChannelData(0);
|
|
14
|
+
for (let i = 0; i < int16.length; i++) channelData[i] = int16[i] / 32768;
|
|
15
|
+
return audioBuffer;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class AudioPlayback {
|
|
19
|
+
isPlaying = false;
|
|
20
|
+
private context: AudioContext | null = null;
|
|
21
|
+
private gain: GainNode | null = null;
|
|
22
|
+
private queue: ArrayBuffer[] = [];
|
|
23
|
+
private nextTime = 0;
|
|
24
|
+
private processing = false;
|
|
25
|
+
private active = new Set<AudioBufferSourceNode>();
|
|
26
|
+
private started = false;
|
|
27
|
+
private lastChunk = false;
|
|
28
|
+
|
|
29
|
+
/** `onChange` hears each change of `isPlaying`. */
|
|
30
|
+
constructor(private readonly onChange: () => void = () => {}) {}
|
|
31
|
+
|
|
32
|
+
private setPlaying(v: boolean): void {
|
|
33
|
+
if (this.isPlaying === v) return;
|
|
34
|
+
this.isPlaying = v;
|
|
35
|
+
this.onChange();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
private audioContext(): AudioContext {
|
|
39
|
+
if (!this.context) {
|
|
40
|
+
this.context = new AudioContext({ sampleRate: 24000 });
|
|
41
|
+
this.gain = this.context.createGain();
|
|
42
|
+
this.gain.connect(this.context.destination);
|
|
43
|
+
}
|
|
44
|
+
if (this.context.state === "suspended") void this.context.resume();
|
|
45
|
+
return this.context;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
private process(): void {
|
|
49
|
+
if (this.processing || this.queue.length === 0) return;
|
|
50
|
+
this.processing = true;
|
|
51
|
+
const ctx = this.audioContext();
|
|
52
|
+
const gain = this.gain!;
|
|
53
|
+
while (this.queue.length > 0) {
|
|
54
|
+
const data = this.queue.shift()!;
|
|
55
|
+
try {
|
|
56
|
+
const audioBuffer = decodeLPCM(ctx, data);
|
|
57
|
+
const now = ctx.currentTime;
|
|
58
|
+
// A CUSHION WHEN THE QUEUE IS EMPTY: a vendor streaming at speaking pace (GPT-Live)
|
|
59
|
+
// sends each frame just as the last ends, so a late frame would leave a gap, heard as a
|
|
60
|
+
// click. Starting a little ahead absorbs it; once playing, frames join end to end.
|
|
61
|
+
const startTime = this.nextTime > now ? this.nextTime : now + START_LEAD_SECONDS;
|
|
62
|
+
const source = ctx.createBufferSource();
|
|
63
|
+
source.buffer = audioBuffer;
|
|
64
|
+
source.connect(gain);
|
|
65
|
+
this.active.add(source);
|
|
66
|
+
if (!this.started) {
|
|
67
|
+
this.started = true;
|
|
68
|
+
this.setPlaying(true);
|
|
69
|
+
}
|
|
70
|
+
source.start(startTime);
|
|
71
|
+
this.nextTime = startTime + audioBuffer.duration;
|
|
72
|
+
source.onended = () => {
|
|
73
|
+
this.active.delete(source);
|
|
74
|
+
if (this.lastChunk && this.active.size === 0 && this.queue.length === 0) {
|
|
75
|
+
this.setPlaying(false);
|
|
76
|
+
this.started = false;
|
|
77
|
+
this.lastChunk = false;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
} catch (error) {
|
|
81
|
+
console.error("[AudioPlayback] Failed to decode audio:", error);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
this.processing = false;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
play(audioData: ArrayBuffer): void {
|
|
88
|
+
this.queue.push(audioData);
|
|
89
|
+
this.process();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
stopAll(): void {
|
|
93
|
+
this.active.forEach((source) => {
|
|
94
|
+
try {
|
|
95
|
+
source.stop();
|
|
96
|
+
} catch {
|
|
97
|
+
// already stopped
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
this.active.clear();
|
|
101
|
+
this.queue = [];
|
|
102
|
+
this.nextTime = 0;
|
|
103
|
+
this.started = false;
|
|
104
|
+
this.processing = false;
|
|
105
|
+
this.lastChunk = false;
|
|
106
|
+
this.setPlaying(false);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** No more chunks are coming: `isPlaying` clears when the last one finishes. */
|
|
110
|
+
markAsLastChunk(): void {
|
|
111
|
+
this.lastChunk = true;
|
|
112
|
+
if (this.active.size === 0 && this.queue.length === 0 && this.started) {
|
|
113
|
+
this.setPlaying(false);
|
|
114
|
+
this.started = false;
|
|
115
|
+
this.lastChunk = false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Wake the speaker, inside a click: a browser plays nothing it was not woken for. */
|
|
120
|
+
unlock(): void {
|
|
121
|
+
this.audioContext();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
close(): void {
|
|
125
|
+
this.stopAll();
|
|
126
|
+
void this.context?.close();
|
|
127
|
+
this.context = null;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The audio lane: a binary WebSocket on `/ws/gravity` for real-time audio. Binary PCM Int16
|
|
3
|
+
* 16 kHz goes up, binary LPCM Int16 24 kHz comes back, plus JSON control messages for the
|
|
4
|
+
* audio-state events. The audio-state events ride this lane WITH the frames so they stay
|
|
5
|
+
* frame-synchronized; they never go on the MCP component stream (docs/VOICE_STREAMING_GUIDE).
|
|
6
|
+
*/
|
|
7
|
+
import type { ControlMessage, AudioStateEvent } from "./types";
|
|
8
|
+
|
|
9
|
+
export interface RealtimeSocketOptions {
|
|
10
|
+
/** Conversation ID for the WebSocket session. */
|
|
11
|
+
sessionId: string;
|
|
12
|
+
userId?: string;
|
|
13
|
+
chatId?: string;
|
|
14
|
+
/** WebSocket base URL (e.g. ws://localhost:4100). Defaults to the current origin. */
|
|
15
|
+
wsUrl?: string;
|
|
16
|
+
/** A fresh access token, asked for on each connect. */
|
|
17
|
+
getAccessToken?: () => Promise<string | null> | string | null;
|
|
18
|
+
onAudioReceived?: (audioData: ArrayBuffer) => void;
|
|
19
|
+
onConnectionChange?: (connected: boolean) => void;
|
|
20
|
+
onControlMessage?: (message: ControlMessage) => void;
|
|
21
|
+
onAudioState?: (event: AudioStateEvent) => void;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class RealtimeSocket {
|
|
25
|
+
private ws: WebSocket | null = null;
|
|
26
|
+
isConnected = false;
|
|
27
|
+
|
|
28
|
+
constructor(private readonly o: RealtimeSocketOptions) {}
|
|
29
|
+
|
|
30
|
+
private async url(): Promise<string> {
|
|
31
|
+
const { wsUrl, getAccessToken } = this.o;
|
|
32
|
+
let baseUrl: string;
|
|
33
|
+
if (wsUrl) baseUrl = `${wsUrl}/ws/gravity`;
|
|
34
|
+
else {
|
|
35
|
+
const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
|
|
36
|
+
baseUrl = `${protocol}//${window.location.host}/ws/gravity`;
|
|
37
|
+
}
|
|
38
|
+
let token: string | null = null;
|
|
39
|
+
if (getAccessToken) {
|
|
40
|
+
try {
|
|
41
|
+
token = await getAccessToken();
|
|
42
|
+
} catch (error) {
|
|
43
|
+
console.warn("[RealtimeWebSocket] Failed to get access token:", error);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return token ? `${baseUrl}?token=${encodeURIComponent(token)}` : baseUrl;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async connect(): Promise<void> {
|
|
50
|
+
if (this.ws?.readyState === WebSocket.OPEN) return;
|
|
51
|
+
if (this.ws?.readyState === WebSocket.CONNECTING) {
|
|
52
|
+
return new Promise((resolve) => {
|
|
53
|
+
const check = setInterval(() => {
|
|
54
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
55
|
+
clearInterval(check);
|
|
56
|
+
resolve();
|
|
57
|
+
}
|
|
58
|
+
}, 50);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const url = await this.url();
|
|
62
|
+
console.log("[RealtimeWebSocket] Connecting to:", url.replace(/token=[^&]+/, "token=***"));
|
|
63
|
+
const { sessionId, userId, chatId, onAudioReceived, onConnectionChange, onControlMessage, onAudioState } = this.o;
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
const ws = new WebSocket(url);
|
|
66
|
+
ws.binaryType = "arraybuffer";
|
|
67
|
+
const timeout = setTimeout(() => reject(new Error("WebSocket connection timeout")), 10000);
|
|
68
|
+
ws.onopen = () => {
|
|
69
|
+
clearTimeout(timeout);
|
|
70
|
+
ws.send(JSON.stringify({ type: "INIT_SESSION", conversationId: sessionId, userId: userId || "anonymous", chatId: chatId || `chat_${sessionId}` }));
|
|
71
|
+
this.isConnected = true;
|
|
72
|
+
onConnectionChange?.(true);
|
|
73
|
+
resolve();
|
|
74
|
+
};
|
|
75
|
+
ws.onmessage = (event) => {
|
|
76
|
+
if (event.data instanceof ArrayBuffer) {
|
|
77
|
+
onAudioReceived?.(event.data);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const message: ControlMessage = JSON.parse(event.data as string);
|
|
82
|
+
onControlMessage?.(message);
|
|
83
|
+
// Only control-channel state events (`message.state`); `message.audioState` is
|
|
84
|
+
// per-chunk metadata on audio publishes.
|
|
85
|
+
if (message.state && onAudioState) onAudioState({ state: message.state, metadata: message.metadata });
|
|
86
|
+
} catch (error) {
|
|
87
|
+
console.error("[RealtimeWebSocket] Failed to parse control message:", error);
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
ws.onerror = (error) => {
|
|
91
|
+
clearTimeout(timeout);
|
|
92
|
+
console.error("[RealtimeWebSocket] Error:", error);
|
|
93
|
+
reject(error);
|
|
94
|
+
};
|
|
95
|
+
ws.onclose = () => {
|
|
96
|
+
this.isConnected = false;
|
|
97
|
+
onConnectionChange?.(false);
|
|
98
|
+
};
|
|
99
|
+
this.ws = ws;
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
disconnect(): void {
|
|
104
|
+
if (this.ws) {
|
|
105
|
+
this.ws.close();
|
|
106
|
+
this.ws = null;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
sendAudio(audioData: ArrayBuffer): void {
|
|
111
|
+
if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(audioData);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
sendControl(type: string, data?: Record<string, unknown>): void {
|
|
115
|
+
if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify({ type, ...data }));
|
|
116
|
+
}
|
|
117
|
+
}
|
package/vendor/sdk/lib/style.ts
CHANGED
|
@@ -4,8 +4,16 @@
|
|
|
4
4
|
* px/rem/em, no recipes); every value lives in design/styles and is served. See
|
|
5
5
|
* FRAMEWORK.md and render.tsx's header. Enforced by test/golden-rule.test.mjs.
|
|
6
6
|
*/
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
7
|
+
import { resolveValue } from "./core/actions";
|
|
8
|
+
import { matches } from "./core/conditions";
|
|
9
|
+
import type { ResolvedTheme, StyleVariant } from "./core/types";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A resolved style: CSS property names in camelCase (`flexDirection`), each a CSS value. A
|
|
13
|
+
* number is a length in px unless the property takes plain numbers (`zIndex`, `flex`,
|
|
14
|
+
* `fontWeight`, `lineHeight`, `opacity`); the element layer applies that rule.
|
|
15
|
+
*/
|
|
16
|
+
export type CssStyle = { [property: string]: string | number | undefined };
|
|
9
17
|
|
|
10
18
|
|
|
11
19
|
// Resolution cache: style objects come from CACHED definitions, so they are
|
|
@@ -14,7 +22,7 @@ import { resolveValue, matches, type ResolvedTheme, type StyleVariant } from "./
|
|
|
14
22
|
// styles ({{field}} bindings or `when` variants) resolve against the live data scope
|
|
15
23
|
// and bypass the cache; the one-time dependency check is itself memoized per object.
|
|
16
24
|
// WeakMaps throughout — nothing here pins a def or theme in memory.
|
|
17
|
-
const styleCache = new WeakMap<ResolvedTheme, WeakMap<Record<string, unknown>,
|
|
25
|
+
const styleCache = new WeakMap<ResolvedTheme, WeakMap<Record<string, unknown>, CssStyle>>();
|
|
18
26
|
const dataDependent = new WeakMap<Record<string, unknown>, boolean>();
|
|
19
27
|
// Stable default so styleless nodes hit the cache instead of keying a fresh {} per call.
|
|
20
28
|
const EMPTY_STYLE: Record<string, unknown> = Object.freeze({});
|
|
@@ -36,7 +44,7 @@ function isDataDependent(s: Record<string, unknown>): boolean {
|
|
|
36
44
|
* charts be DATA (Box + Each), not a primitive: the producer supplies the proportion,
|
|
37
45
|
* the SDK authors nothing. (Same `{{...}}` resolver the action vocab uses.)
|
|
38
46
|
*/
|
|
39
|
-
export function styleToCss(s: Record<string, unknown> = EMPTY_STYLE, theme: ResolvedTheme, data?: Record<string, unknown>):
|
|
47
|
+
export function styleToCss(s: Record<string, unknown> = EMPTY_STYLE, theme: ResolvedTheme, data?: Record<string, unknown>): CssStyle {
|
|
40
48
|
if (!data || !isDataDependent(s)) {
|
|
41
49
|
let byStyle = styleCache.get(theme);
|
|
42
50
|
if (!byStyle) {
|
|
@@ -52,7 +60,7 @@ export function styleToCss(s: Record<string, unknown> = EMPTY_STYLE, theme: Reso
|
|
|
52
60
|
return computeCss(s, theme, data);
|
|
53
61
|
}
|
|
54
62
|
|
|
55
|
-
function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Record<string, unknown>):
|
|
63
|
+
function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Record<string, unknown>): CssStyle {
|
|
56
64
|
if (data) {
|
|
57
65
|
const resolved: Record<string, unknown> = {};
|
|
58
66
|
for (const [k, v] of Object.entries(s)) resolved[k] = typeof v === "string" && v.includes("{{") ? resolveValue(v, data) : v;
|
|
@@ -71,7 +79,7 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
71
79
|
s = merged;
|
|
72
80
|
}
|
|
73
81
|
}
|
|
74
|
-
const css:
|
|
82
|
+
const css: CssStyle = {};
|
|
75
83
|
// EVERY bucket read below is optional-chained (`theme.x?.[k] ?? k`), never `theme.x[k]`.
|
|
76
84
|
// A bucket is absent whenever the theme was built by an OLDER base than the SDK reading
|
|
77
85
|
// it — a version skew that is normal for an installed Studio or a served app, since the
|
|
@@ -86,40 +94,40 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
86
94
|
// A dimension is a scale step ("8") OR a named PAGE WIDTH ("reading"). Both live on
|
|
87
95
|
// the same scale — `layout.*` are aliases onto it — so one lookup order covers both.
|
|
88
96
|
const dim = (v: unknown) =>
|
|
89
|
-
theme.space?.[v as string] ?? theme.layout?.[v as string] ?? (v === "full" ? "100%" : (v as
|
|
97
|
+
theme.space?.[v as string] ?? theme.layout?.[v as string] ?? (v === "full" ? "100%" : (v as string));
|
|
90
98
|
|
|
91
99
|
if (s.width != null) css.width = dim(s.width);
|
|
92
100
|
if (s.height != null) css.height = dim(s.height);
|
|
93
|
-
if (s.maxWidth != null) css.maxWidth = dim(s.maxWidth)
|
|
94
|
-
if (s.minWidth != null) css.minWidth = dim(s.minWidth)
|
|
95
|
-
if (s.minHeight != null) css.minHeight = dim(s.minHeight)
|
|
96
|
-
if (s.maxHeight != null) css.maxHeight = dim(s.maxHeight)
|
|
97
|
-
if (s.flex != null) css.flex = s.flex as
|
|
98
|
-
if (s.position) css.position = s.position as
|
|
99
|
-
if (s.inset != null) css.inset = s.inset as
|
|
101
|
+
if (s.maxWidth != null) css.maxWidth = dim(s.maxWidth);
|
|
102
|
+
if (s.minWidth != null) css.minWidth = dim(s.minWidth);
|
|
103
|
+
if (s.minHeight != null) css.minHeight = dim(s.minHeight);
|
|
104
|
+
if (s.maxHeight != null) css.maxHeight = dim(s.maxHeight);
|
|
105
|
+
if (s.flex != null) css.flex = s.flex as string | number;
|
|
106
|
+
if (s.position) css.position = s.position as string;
|
|
107
|
+
if (s.inset != null) css.inset = s.inset as string;
|
|
100
108
|
// Edge offsets (with `position`) — generic positioning, e.g. a bottom-anchored overlay
|
|
101
109
|
// panel that floats above without pushing siblings. Resolve like any dimension (`full`→100%).
|
|
102
|
-
if (s.top != null) css.top = dim(s.top)
|
|
103
|
-
if (s.right != null) css.right = dim(s.right)
|
|
104
|
-
if (s.bottom != null) css.bottom = dim(s.bottom)
|
|
105
|
-
if (s.left != null) css.left = dim(s.left)
|
|
106
|
-
if (s.zIndex != null) css.zIndex = s.zIndex as
|
|
110
|
+
if (s.top != null) css.top = dim(s.top);
|
|
111
|
+
if (s.right != null) css.right = dim(s.right);
|
|
112
|
+
if (s.bottom != null) css.bottom = dim(s.bottom);
|
|
113
|
+
if (s.left != null) css.left = dim(s.left);
|
|
114
|
+
if (s.zIndex != null) css.zIndex = s.zIndex as string | number;
|
|
107
115
|
if (s.direction) {
|
|
108
116
|
css.display = "flex";
|
|
109
|
-
css.flexDirection = s.direction as
|
|
117
|
+
css.flexDirection = s.direction as string;
|
|
110
118
|
}
|
|
111
119
|
// `wrap` — flex-wrap, so a row of content-width items (e.g. FAQ chips, tag pills) sits
|
|
112
120
|
// side-by-side when there's room and wraps to the next line when there isn't. Generic
|
|
113
121
|
// layout vocab (any wrapping flex container); the value ("wrap"/"nowrap") is served/literal.
|
|
114
|
-
if (s.wrap) css.flexWrap = s.wrap as
|
|
122
|
+
if (s.wrap) css.flexWrap = s.wrap as string;
|
|
115
123
|
if (s.gap) css.gap = space(s.gap as string);
|
|
116
124
|
if (s.padding) css.padding = Array.isArray(s.padding) ? (s.padding as string[]).map(space).join(" ") : space(s.padding as string);
|
|
117
125
|
if (s.margin) css.margin = Array.isArray(s.margin) ? (s.margin as string[]).map(space).join(" ") : space(s.margin as string);
|
|
118
|
-
if (s.display) css.display = s.display as
|
|
126
|
+
if (s.display) css.display = s.display as string;
|
|
119
127
|
// `container` — establish a container-query context (e.g. "inline-size") so descendants
|
|
120
128
|
// can respond to THIS element's width via `hideBelow` (component-relative responsiveness,
|
|
121
129
|
// correct for SDUI: the component reacts to its slot, not the page viewport). Value served.
|
|
122
|
-
if (s.container)
|
|
130
|
+
if (s.container) css.containerType = s.container as string;
|
|
123
131
|
// `columns` → an equal-width CSS grid (generic layout; the welcome card grid uses it).
|
|
124
132
|
// A MAP value ({mobile, tablet, ...}) is the RESPONSIVE form — handled in
|
|
125
133
|
// primitives.tsx as an injected rule, since inline styles cannot express the
|
|
@@ -132,25 +140,25 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
132
140
|
// it — which is exactly the direction the stacking rule runs (primitives.tsx). Without
|
|
133
141
|
// this a grid would need a `container` ancestor to stack, and would silently not stack
|
|
134
142
|
// wherever one was missing.
|
|
135
|
-
|
|
143
|
+
css.containerType = "inline-size";
|
|
136
144
|
// Pack rows at the top. Without this, a columns-grid inside a flex:1 panel
|
|
137
145
|
// inherits CSS's default align-content:stretch and distributes the panel's
|
|
138
146
|
// leftover height BETWEEN rows — the phantom vertical gap between card rows.
|
|
139
147
|
css.alignContent = "start";
|
|
140
148
|
}
|
|
141
|
-
if (s.cursor) css.cursor = s.cursor as
|
|
149
|
+
if (s.cursor) css.cursor = s.cursor as string;
|
|
142
150
|
// `span` — how many of the parent grid's columns this child covers: on a 12-column
|
|
143
151
|
// grid, 6 is a half and 3 is a quarter. The grid owns the gaps, so spans never overflow
|
|
144
152
|
// the way percentage widths in a flex row do. Below the stacking width the child takes
|
|
145
153
|
// the full row instead (the rule injected in primitives.tsx).
|
|
146
154
|
if (s.span) css.gridColumn = `span ${s.span}`;
|
|
147
|
-
if (s.align) css.alignItems = s.align === "start" ? "flex-start" : (s.align as
|
|
148
|
-
if (s.justify) css.justifyContent = s.justify as
|
|
155
|
+
if (s.align) css.alignItems = s.align === "start" ? "flex-start" : (s.align as string);
|
|
156
|
+
if (s.justify) css.justifyContent = s.justify as string;
|
|
149
157
|
// Explicit text alignment (inherited by children) — needed because a Button's
|
|
150
158
|
// user-agent stylesheet centers text, which flex `align` can't undo on wrapped lines.
|
|
151
|
-
if (s.textAlign) css.textAlign = (s.textAlign === "start" ? "left" : s.textAlign) as
|
|
159
|
+
if (s.textAlign) css.textAlign = (s.textAlign === "start" ? "left" : s.textAlign) as string;
|
|
152
160
|
// Generic transform passthrough (values are DATA, e.g. a hover lift) — the SDK maps, never authors.
|
|
153
|
-
if (s.transform) css.transform = s.transform as
|
|
161
|
+
if (s.transform) css.transform = s.transform as string;
|
|
154
162
|
if (s.background) css.background = theme.color?.[s.background as string] ?? (s.background as string);
|
|
155
163
|
// `radial` — a neutral dial/gauge fill: a `fill` token sweeps from 0 to `at` (a value
|
|
156
164
|
// or {{field}} binding, e.g. "72%"), the rest is the `track` token. The WEB realization
|
|
@@ -180,7 +188,7 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
180
188
|
if (s.borderBottom) css.borderBottom = borderVal(s.borderBottom as string);
|
|
181
189
|
if (s.borderLeft) css.borderLeft = borderVal(s.borderLeft as string);
|
|
182
190
|
// `outline` — pass-through (e.g. "none" to drop the input focus ring inside a pill).
|
|
183
|
-
if (s.outline != null) css.outline = s.outline as
|
|
191
|
+
if (s.outline != null) css.outline = s.outline as string;
|
|
184
192
|
if (s.shadow) css.boxShadow = theme.shadow?.[s.shadow as string] ?? (s.shadow as string);
|
|
185
193
|
const rad = (v: string) => theme.radius?.[v] ?? v;
|
|
186
194
|
if (s.radius) css.borderRadius = rad(s.radius as string);
|
|
@@ -189,22 +197,22 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
189
197
|
if (s.radiusTopRight) css.borderTopRightRadius = rad(s.radiusTopRight as string);
|
|
190
198
|
if (s.radiusBottomLeft) css.borderBottomLeftRadius = rad(s.radiusBottomLeft as string);
|
|
191
199
|
if (s.radiusBottomRight) css.borderBottomRightRadius = rad(s.radiusBottomRight as string);
|
|
192
|
-
if (s.overflow) css.overflow = s.overflow as
|
|
193
|
-
if (s.fit) css.objectFit = s.fit as
|
|
200
|
+
if (s.overflow) css.overflow = s.overflow as string;
|
|
201
|
+
if (s.fit) css.objectFit = s.fit as string;
|
|
194
202
|
if (s.font && theme.text?.[s.font as string]) Object.assign(css, theme.text[s.font as string]);
|
|
195
203
|
// `weight` — a SERVED token like every other value here. This was the one scale the
|
|
196
204
|
// SDK owned outright: a literal four-name map, which meant the foundation's own
|
|
197
205
|
// `light`/`black` were unreachable and an org that added a weight to its tokens got
|
|
198
206
|
// nothing at all (the unknown word reached CSS, CSS dropped the declaration, and the
|
|
199
207
|
// text kept whatever it inherited — silently). A raw number still passes through.
|
|
200
|
-
if (s.weight) css.fontWeight = (theme.weight?.[s.weight as string] ?? s.weight) as
|
|
208
|
+
if (s.weight) css.fontWeight = (theme.weight?.[s.weight as string] ?? s.weight) as string | number;
|
|
201
209
|
// `lineHeight` — override the text style's line-height (e.g. tighten a single-line
|
|
202
210
|
// card title). Applied AFTER `font` so it wins. Resolves a lineHeight token (tight/…).
|
|
203
|
-
if (s.lineHeight != null) css.lineHeight = (theme.lineHeight?.[s.lineHeight as string] ?? s.lineHeight) as
|
|
211
|
+
if (s.lineHeight != null) css.lineHeight = (theme.lineHeight?.[s.lineHeight as string] ?? s.lineHeight) as string | number;
|
|
204
212
|
if (s.color) css.color = theme.color?.[s.color as string] ?? (s.color as string);
|
|
205
213
|
// `transition` — smooth state changes (e.g. hover). The value is authored in the
|
|
206
214
|
// definition (e.g. "all 150ms ease"); the SDK passes it through, authoring nothing.
|
|
207
|
-
if (s.transition) css.transition = s.transition as
|
|
215
|
+
if (s.transition) css.transition = s.transition as string;
|
|
208
216
|
// Animation — GENERIC vocab. `animation` names a served keyframe (theme.keyframes,
|
|
209
217
|
// injected once via keyframesCss); timing comes from the definition. The SDK authors
|
|
210
218
|
// no motion values, and there is NO per-animation primitive — any animated UX is data.
|
|
@@ -217,12 +225,12 @@ function computeCss(s: Record<string, unknown>, theme: ResolvedTheme, data?: Rec
|
|
|
217
225
|
}
|
|
218
226
|
|
|
219
227
|
/**
|
|
220
|
-
* Serialize a resolved
|
|
228
|
+
* Serialize a resolved CssStyle object → a CSS declaration string ("a-b:v;…").
|
|
221
229
|
* `important` is needed for injected state rules (hover/active): the base style is
|
|
222
230
|
* applied INLINE, and an inline style beats any class selector — so a `:hover` rule
|
|
223
231
|
* can only override it with `!important`.
|
|
224
232
|
*/
|
|
225
|
-
export function cssDecls(css:
|
|
233
|
+
export function cssDecls(css: CssStyle, important = false): string {
|
|
226
234
|
const flag = important ? " !important" : "";
|
|
227
235
|
return Object.entries(css)
|
|
228
236
|
.map(([k, v]) => `${k.replace(/[A-Z]/g, (m) => "-" + m.toLowerCase())}:${v}${flag}`)
|