@alia.onl/sdk 3.1.1 → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/alia.onl-sdk-3.1.3.tgz +0 -0
- package/package.json +80 -69
- package/src/animations.ts +6 -0
- package/src/components/AliaChatContent.tsx +215 -253
- package/src/components/AliaChatMessageList.tsx +9 -4
- package/src/components/AliaChatScreen.tsx +28 -42
- package/src/components/AliaChatSheet.tsx +40 -55
- package/src/components/AliaMark.tsx +144 -0
- package/src/components/AliaWelcomeMessage.tsx +5 -5
- package/src/components/Markdown.tsx +18 -18
- package/src/components/PlanPreviewCard.tsx +3 -2
- package/src/components/ThinkingIndicator.tsx +6 -27
- package/src/components/ui/chat-text-input.tsx +4 -3
- package/src/components/ui/dropdown-menu.web.tsx +9 -8
- package/src/components/ui/prompt-input/autocomplete.tsx +1 -1
- package/src/components/ui/prompt-input/context.tsx +1 -1
- package/src/components/ui/prompt-input/prompt-input.tsx +3 -1
- package/src/components/voice/AudioWaveVisualizer.tsx +27 -10
- package/src/components/voice/VoiceOverlay.tsx +37 -9
- package/src/hooks/useAliaChat.ts +6 -6
- package/src/hooks/useAmbientWave.ts +142 -0
- package/src/hooks/useAudioLevelMonitor.ts +6 -6
- package/src/hooks/useDocumentPicker.ts +1 -1
- package/src/hooks/useImagePicker.ts +1 -1
- package/src/hooks/useSoundEffects.ts +6 -5
- package/src/hooks/useSpeechToText.ts +21 -7
- package/src/hooks/useTTS.ts +6 -4
- package/src/hooks/useVoiceRoom.ts +52 -7
- package/src/index.ts +4 -2
- package/src/lib/content-utils.ts +18 -6
- package/src/lib/utils.ts +11 -0
- package/src/optional-deps.d.ts +76 -15
- package/src/types.ts +4 -4
- package/src/components/AliaFace.tsx +0 -452
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
type TextInputKeyPressEventData,
|
|
8
8
|
type NativeSyntheticEvent as RNSyntheticEvent,
|
|
9
9
|
type TextInputContentSizeChangeEventData,
|
|
10
|
+
type TextStyle,
|
|
10
11
|
} from "react-native";
|
|
11
12
|
import { cn } from "../../lib/utils";
|
|
12
13
|
|
|
@@ -144,12 +145,12 @@ const ChatTextInput = React.forwardRef<TextInput, ChatTextInputProps>(
|
|
|
144
145
|
scrollEnabled={fillContainer || props.multiline}
|
|
145
146
|
style={[
|
|
146
147
|
style,
|
|
147
|
-
!fillContainer && props.multiline && !disableAutoHeight && {
|
|
148
|
+
!fillContainer && props.multiline && !disableAutoHeight && ({
|
|
148
149
|
minHeight,
|
|
149
150
|
maxHeight,
|
|
150
|
-
overflow: 'auto'
|
|
151
|
+
overflow: 'auto',
|
|
151
152
|
...(Platform.OS === 'web' ? { fieldSizing: 'content' } : {}),
|
|
152
|
-
},
|
|
153
|
+
} as TextStyle & { overflow: 'auto'; fieldSizing?: string }),
|
|
153
154
|
fillContainer && { flex: 1, height: '100%' },
|
|
154
155
|
]}
|
|
155
156
|
{...props}
|
|
@@ -4,6 +4,7 @@ import { cn } from "../../lib/utils";
|
|
|
4
4
|
import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu";
|
|
5
5
|
import { CheckIcon, ChevronRightIcon } from "@radix-ui/react-icons";
|
|
6
6
|
import * as React from "react";
|
|
7
|
+
import type { LucideIcon } from "lucide-react-native";
|
|
7
8
|
import {
|
|
8
9
|
Star, Pencil, Trash2, Share2, Download, Settings, HelpCircle,
|
|
9
10
|
Image, FileText, Search, ShoppingBag, MoreHorizontal, ExternalLink,
|
|
@@ -12,7 +13,7 @@ import {
|
|
|
12
13
|
} from "lucide-react-native";
|
|
13
14
|
|
|
14
15
|
// Map iOS SF Symbol names to Lucide icons for web rendering
|
|
15
|
-
const SF_SYMBOL_MAP: Record<string,
|
|
16
|
+
const SF_SYMBOL_MAP: Record<string, LucideIcon> = {
|
|
16
17
|
"star": Star, "star.fill": Star, "pencil": Pencil, "trash": Trash2,
|
|
17
18
|
"square.and.arrow.up": Share2, "arrow.down.doc": Download,
|
|
18
19
|
"gearshape": Settings, "questionmark.circle": HelpCircle,
|
|
@@ -142,8 +143,8 @@ const DropdownMenuSubContent = React.forwardRef<
|
|
|
142
143
|
)}
|
|
143
144
|
style={{
|
|
144
145
|
boxShadow: '0 4px 6px -1px rgba(0,0,0,0.1), 0 2px 4px -2px rgba(0,0,0,0.1)',
|
|
145
|
-
border: '1px solid
|
|
146
|
-
...style
|
|
146
|
+
border: '1px solid var(--border)',
|
|
147
|
+
...style,
|
|
147
148
|
}}
|
|
148
149
|
{...props}
|
|
149
150
|
/>
|
|
@@ -168,8 +169,8 @@ const DropdownMenuContent = React.forwardRef<
|
|
|
168
169
|
)}
|
|
169
170
|
style={{
|
|
170
171
|
boxShadow: '0 4px 6px -1px rgba(0,0,0,0.1), 0 2px 4px -2px rgba(0,0,0,0.1)',
|
|
171
|
-
border: '1px solid
|
|
172
|
-
...style
|
|
172
|
+
border: '1px solid var(--border)',
|
|
173
|
+
...style,
|
|
173
174
|
}}
|
|
174
175
|
{...props}
|
|
175
176
|
/>
|
|
@@ -198,7 +199,7 @@ const DropdownMenuItem = React.forwardRef<
|
|
|
198
199
|
) => (
|
|
199
200
|
<DropdownMenuPrimitive.Item
|
|
200
201
|
ref={ref}
|
|
201
|
-
onSelect={(e:
|
|
202
|
+
onSelect={(e: Event) => {
|
|
202
203
|
onSelect?.(e);
|
|
203
204
|
if (shouldDismissMenuOnSelect === false) {
|
|
204
205
|
e.preventDefault();
|
|
@@ -244,7 +245,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<
|
|
|
244
245
|
) => (
|
|
245
246
|
<DropdownMenuPrimitive.CheckboxItem
|
|
246
247
|
ref={ref}
|
|
247
|
-
onSelect={(e:
|
|
248
|
+
onSelect={(e: Event) => {
|
|
248
249
|
const current =
|
|
249
250
|
value === true ? "on" : value === false ? "off" : value;
|
|
250
251
|
const next = current === "on" ? "off" : "on";
|
|
@@ -305,7 +306,7 @@ DropdownMenuSeparator.displayName =
|
|
|
305
306
|
|
|
306
307
|
export const ItemIcon = React.forwardRef<
|
|
307
308
|
HTMLSpanElement,
|
|
308
|
-
React.HTMLProps<HTMLSpanElement> & { ios?: { name?: string; [key: string]:
|
|
309
|
+
React.HTMLProps<HTMLSpanElement> & { ios?: { name?: string; [key: string]: unknown }; androidIconName?: string }
|
|
309
310
|
>(({ className, ios, androidIconName, children, ...props }, ref) => {
|
|
310
311
|
const IconComponent = ios?.name ? SF_SYMBOL_MAP[ios.name] : null;
|
|
311
312
|
return (
|
|
@@ -122,7 +122,7 @@ export function PromptInputAutocomplete({
|
|
|
122
122
|
// Register/unregister the key handler based on completions
|
|
123
123
|
useEffect(() => {
|
|
124
124
|
if (completions.length > 0) {
|
|
125
|
-
|
|
125
|
+
setHandleCompletionKey(() => handleKey);
|
|
126
126
|
} else {
|
|
127
127
|
setHandleCompletionKey(null);
|
|
128
128
|
}
|
|
@@ -35,7 +35,7 @@ export type PromptInputContextType = {
|
|
|
35
35
|
removeAttachment: (id: string) => void;
|
|
36
36
|
updateAttachment: (id: string, updates: Partial<Attachment>) => void;
|
|
37
37
|
handleCompletionKey: ((key: string) => boolean) | null;
|
|
38
|
-
setHandleCompletionKey: (
|
|
38
|
+
setHandleCompletionKey: React.Dispatch<React.SetStateAction<((key: string) => boolean) | null>>;
|
|
39
39
|
// Injectable autocomplete hooks
|
|
40
40
|
useSuggestions?: (query: string) => { data: Completion[] | undefined; isLoading: boolean };
|
|
41
41
|
onSuggestionUsed?: (suggestionId: string) => void;
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
View,
|
|
4
4
|
Pressable,
|
|
5
5
|
type TextInput as RNTextInput,
|
|
6
|
+
type ViewStyle,
|
|
6
7
|
} from "react-native";
|
|
7
8
|
import { KeyboardAvoidingView } from "../../../lib/keyboard";
|
|
8
9
|
import { Maximize2, Minimize2 } from "lucide-react-native";
|
|
@@ -253,7 +254,8 @@ export function PromptInput({
|
|
|
253
254
|
{showFullscreen && (
|
|
254
255
|
<View
|
|
255
256
|
style={{
|
|
256
|
-
position:
|
|
257
|
+
// `position: 'fixed'` is a web-only value (NativeWind) not in RN's ViewStyle union.
|
|
258
|
+
position: "fixed" as unknown as ViewStyle["position"],
|
|
257
259
|
top: 0,
|
|
258
260
|
left: 0,
|
|
259
261
|
right: 0,
|
|
@@ -16,10 +16,14 @@ import type { AgentState } from '../../types';
|
|
|
16
16
|
const AnimatedPath = Animated.createAnimatedComponent(Path);
|
|
17
17
|
|
|
18
18
|
/** Cross-platform blur: CSS string on web, RN array syntax on native */
|
|
19
|
-
|
|
19
|
+
type BlurStyle =
|
|
20
|
+
| { filter: string; WebkitFilter: string }
|
|
21
|
+
| { filter: Array<{ blur: number }> };
|
|
22
|
+
|
|
23
|
+
function blurStyle(radius: number): BlurStyle {
|
|
20
24
|
return Platform.OS === 'web'
|
|
21
|
-
?
|
|
22
|
-
:
|
|
25
|
+
? { filter: `blur(${radius}px)`, WebkitFilter: `blur(${radius}px)` }
|
|
26
|
+
: { filter: [{ blur: radius }] };
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
const GLOW_HEIGHT = 350;
|
|
@@ -175,9 +179,21 @@ const BLOB_COLORS: Record<AgentState, string[]> = {
|
|
|
175
179
|
};
|
|
176
180
|
|
|
177
181
|
// Derive wave & blob palettes from a hex color
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
182
|
+
/** Parse a CSS color string (#rgb, #rrggbb, rgb(...), rgba(...)) to [r,g,b]; null if unrecognized. */
|
|
183
|
+
function parseColor(input: string): [number, number, number] | null {
|
|
184
|
+
const s = input.trim();
|
|
185
|
+
if (s.startsWith('#')) {
|
|
186
|
+
const h = s.slice(1);
|
|
187
|
+
const expanded = h.length === 3 ? h[0] + h[0] + h[1] + h[1] + h[2] + h[2] : h;
|
|
188
|
+
if (expanded.length !== 6) return null;
|
|
189
|
+
const r = parseInt(expanded.slice(0, 2), 16);
|
|
190
|
+
const g = parseInt(expanded.slice(2, 4), 16);
|
|
191
|
+
const b = parseInt(expanded.slice(4, 6), 16);
|
|
192
|
+
return Number.isNaN(r) || Number.isNaN(g) || Number.isNaN(b) ? null : [r, g, b];
|
|
193
|
+
}
|
|
194
|
+
// rgb()/rgba() — comma OR space separated (rgb(r, g, b) and modern rgb(r g b / a)); alpha ignored
|
|
195
|
+
const match = s.match(/^rgba?\(\s*(\d+)[\s,]+(\d+)[\s,]+(\d+)/i);
|
|
196
|
+
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
|
|
181
197
|
}
|
|
182
198
|
|
|
183
199
|
function darken(rgb: [number, number, number], factor: number): [number, number, number] {
|
|
@@ -192,8 +208,9 @@ function lighten(rgb: [number, number, number], factor: number): [number, number
|
|
|
192
208
|
];
|
|
193
209
|
}
|
|
194
210
|
|
|
195
|
-
function deriveWavePalette(
|
|
196
|
-
const rgb =
|
|
211
|
+
function deriveWavePalette(color: string, isDark: boolean): { waves: string[]; blobs: string[] } | null {
|
|
212
|
+
const rgb = parseColor(color);
|
|
213
|
+
if (!rgb) return null;
|
|
197
214
|
// Dark mode: lighter/brighter waves visible against dark bg
|
|
198
215
|
// Light mode: darker waves visible against light bg
|
|
199
216
|
const layer0 = isDark ? darken(rgb, 0.55) : darken(rgb, 0.3);
|
|
@@ -219,7 +236,7 @@ function deriveWavePalette(hex: string, isDark: boolean): { waves: string[]; blo
|
|
|
219
236
|
interface AudioWaveVisualizerProps {
|
|
220
237
|
waveAmplitude: SharedValue<number>;
|
|
221
238
|
agentState: AgentState;
|
|
222
|
-
isConnected
|
|
239
|
+
isConnected?: boolean;
|
|
223
240
|
/** Theme primary color hex — derives idle/listening wave palette */
|
|
224
241
|
primaryColor?: string;
|
|
225
242
|
/** Whether the app is in dark mode (affects wave brightness) */
|
|
@@ -229,7 +246,7 @@ interface AudioWaveVisualizerProps {
|
|
|
229
246
|
export function AudioWaveVisualizer({
|
|
230
247
|
waveAmplitude,
|
|
231
248
|
agentState = 'idle',
|
|
232
|
-
isConnected,
|
|
249
|
+
isConnected = true,
|
|
233
250
|
primaryColor,
|
|
234
251
|
isDarkMode,
|
|
235
252
|
}: AudioWaveVisualizerProps) {
|
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Subtle
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Subtle wave overlay that renders the AudioWaveVisualizer behind the
|
|
3
|
+
* conversation message list. Always mounted; `intensity` ramps the opacity
|
|
4
|
+
* between a barely-there idle ambient and a prominent speaking state.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import {
|
|
8
|
-
|
|
7
|
+
import Animated, {
|
|
8
|
+
FadeIn,
|
|
9
|
+
FadeOut,
|
|
10
|
+
useSharedValue,
|
|
11
|
+
useAnimatedReaction,
|
|
12
|
+
useAnimatedStyle,
|
|
13
|
+
withTiming,
|
|
14
|
+
} from 'react-native-reanimated';
|
|
9
15
|
import { AudioWaveVisualizer } from './AudioWaveVisualizer';
|
|
10
16
|
import type { SharedValue } from 'react-native-reanimated';
|
|
11
17
|
import type { AgentState } from '../../types';
|
|
@@ -13,12 +19,34 @@ import type { AgentState } from '../../types';
|
|
|
13
19
|
interface VoiceOverlayProps {
|
|
14
20
|
waveAmplitude: SharedValue<number>;
|
|
15
21
|
agentState: AgentState;
|
|
16
|
-
|
|
22
|
+
/** Overlay opacity target (default 0.35); ramps over 500ms when it changes. */
|
|
23
|
+
intensity?: number;
|
|
24
|
+
isConnected?: boolean;
|
|
17
25
|
primaryColor?: string;
|
|
18
26
|
isDarkMode?: boolean;
|
|
19
27
|
}
|
|
20
28
|
|
|
21
|
-
export function VoiceOverlay({
|
|
29
|
+
export function VoiceOverlay({
|
|
30
|
+
waveAmplitude,
|
|
31
|
+
agentState,
|
|
32
|
+
intensity = 0.35,
|
|
33
|
+
isConnected = true,
|
|
34
|
+
primaryColor,
|
|
35
|
+
isDarkMode,
|
|
36
|
+
}: VoiceOverlayProps) {
|
|
37
|
+
// Imperative opacity ramp (web-safe): mapper-started style animations don't
|
|
38
|
+
// tick on reanimated-web, so drive a shared value via useAnimatedReaction.
|
|
39
|
+
const opacity = useSharedValue(intensity);
|
|
40
|
+
useAnimatedReaction(
|
|
41
|
+
() => intensity,
|
|
42
|
+
(target, previous) => {
|
|
43
|
+
if (target === previous) return;
|
|
44
|
+
opacity.value = withTiming(target, { duration: 500 });
|
|
45
|
+
},
|
|
46
|
+
[intensity],
|
|
47
|
+
);
|
|
48
|
+
const animatedStyle = useAnimatedStyle(() => ({ opacity: opacity.value }));
|
|
49
|
+
|
|
22
50
|
return (
|
|
23
51
|
<Animated.View
|
|
24
52
|
entering={FadeIn.duration(400)}
|
|
@@ -33,7 +61,7 @@ export function VoiceOverlay({ waveAmplitude, agentState, isConnected, primaryCo
|
|
|
33
61
|
pointerEvents: 'none',
|
|
34
62
|
}}
|
|
35
63
|
>
|
|
36
|
-
<View style={
|
|
64
|
+
<Animated.View style={animatedStyle}>
|
|
37
65
|
<AudioWaveVisualizer
|
|
38
66
|
waveAmplitude={waveAmplitude}
|
|
39
67
|
agentState={agentState}
|
|
@@ -41,7 +69,7 @@ export function VoiceOverlay({ waveAmplitude, agentState, isConnected, primaryCo
|
|
|
41
69
|
primaryColor={primaryColor}
|
|
42
70
|
isDarkMode={isDarkMode}
|
|
43
71
|
/>
|
|
44
|
-
</View>
|
|
72
|
+
</Animated.View>
|
|
45
73
|
</Animated.View>
|
|
46
74
|
);
|
|
47
75
|
}
|
package/src/hooks/useAliaChat.ts
CHANGED
|
@@ -128,7 +128,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
128
128
|
|
|
129
129
|
/** Apply a tool result to the last assistant message and sync the ref */
|
|
130
130
|
const applyToolResult = useCallback(
|
|
131
|
-
(toolCallId: string, name: string | undefined, output:
|
|
131
|
+
(toolCallId: string, name: string | undefined, output: unknown) => {
|
|
132
132
|
updateAssistant((last) => {
|
|
133
133
|
const invocations = [...(last.toolInvocations || [])];
|
|
134
134
|
const idx = invocations.findIndex((t) => t.toolCallId === toolCallId);
|
|
@@ -380,9 +380,9 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
380
380
|
const toolName = tc.function?.name;
|
|
381
381
|
if (!toolCallId || !toolName) continue;
|
|
382
382
|
|
|
383
|
-
let args:
|
|
383
|
+
let args: Record<string, unknown> | undefined;
|
|
384
384
|
if (tc.function?.arguments) {
|
|
385
|
-
try { args = JSON.parse(tc.function.arguments)
|
|
385
|
+
try { args = JSON.parse(tc.function.arguments) as Record<string, unknown>; } catch { args = { _raw: tc.function.arguments }; }
|
|
386
386
|
}
|
|
387
387
|
|
|
388
388
|
const invocation: ToolInvocation = { toolCallId, toolName, state: 'call', args };
|
|
@@ -411,9 +411,9 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
411
411
|
} finally {
|
|
412
412
|
reader.releaseLock();
|
|
413
413
|
}
|
|
414
|
-
} catch (err:
|
|
415
|
-
if (err
|
|
416
|
-
const errorMessage = err
|
|
414
|
+
} catch (err: unknown) {
|
|
415
|
+
if (err instanceof Error && err.name === 'AbortError') return;
|
|
416
|
+
const errorMessage = err instanceof Error ? err.message : 'Something went wrong';
|
|
417
417
|
setError(errorMessage);
|
|
418
418
|
updateAssistant((last) =>
|
|
419
419
|
!last.content ? { content: "I'm having trouble connecting right now. Please try again." } : {}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { useMemo } from 'react';
|
|
2
|
+
import {
|
|
3
|
+
useSharedValue,
|
|
4
|
+
useDerivedValue,
|
|
5
|
+
useAnimatedReaction,
|
|
6
|
+
withTiming,
|
|
7
|
+
withRepeat,
|
|
8
|
+
withSequence,
|
|
9
|
+
cancelAnimation,
|
|
10
|
+
Easing,
|
|
11
|
+
type SharedValue,
|
|
12
|
+
} from 'react-native-reanimated';
|
|
13
|
+
import { useSTTStore } from './useSpeechToText';
|
|
14
|
+
import type { AgentState } from '../types';
|
|
15
|
+
|
|
16
|
+
export type AmbientWaveMode = 'voice' | 'tts' | 'stt' | 'thinking' | 'idle';
|
|
17
|
+
|
|
18
|
+
export interface AmbientWaveVoice {
|
|
19
|
+
isActive: boolean;
|
|
20
|
+
isConnected: boolean;
|
|
21
|
+
agentState: AgentState;
|
|
22
|
+
waveAmplitude: SharedValue<number>;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface UseAmbientWaveOptions {
|
|
26
|
+
/** Live voice-call amplitude + state (present only while a call is being set up / active). */
|
|
27
|
+
voice?: AmbientWaveVoice;
|
|
28
|
+
/** Whether TTS ("read aloud") is currently playing. */
|
|
29
|
+
isTTSPlaying: boolean;
|
|
30
|
+
/** Simulated TTS amplitude SharedValue (glides to 0 when not playing). */
|
|
31
|
+
ttsWaveAmplitude: SharedValue<number>;
|
|
32
|
+
/** Text-chat generation in flight (thinking or streaming) — gentle wave motion. */
|
|
33
|
+
isGenerating?: boolean;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface UseAmbientWaveResult {
|
|
37
|
+
/** Combined amplitude — max of idle breath, voice, TTS, and STT sources. */
|
|
38
|
+
waveAmplitude: SharedValue<number>;
|
|
39
|
+
/** Palette-driving state for the visualizer. */
|
|
40
|
+
agentState: AgentState;
|
|
41
|
+
/** Overlay opacity target — subtle at idle, prominent while there is speech. */
|
|
42
|
+
intensity: number;
|
|
43
|
+
/** Semantic label for the active source. */
|
|
44
|
+
mode: AmbientWaveMode;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const IDLE_INTENSITY = 0.13;
|
|
48
|
+
const THINKING_INTENSITY = 0.2;
|
|
49
|
+
const ACTIVE_INTENSITY = 0.35;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Combines every wave-amplitude source into one always-on ambient signal for a
|
|
53
|
+
* single persistent overlay. Idle chat breathes subtly; a voice call, TTS
|
|
54
|
+
* playback, or STT recording intensify it. Because every source glides to 0 when
|
|
55
|
+
* inactive, `max(idleBreath, voice, tts, stt)` gives seamless cross-mode handoffs
|
|
56
|
+
* with no remounts.
|
|
57
|
+
*
|
|
58
|
+
* All animation starts are imperative (via `useAnimatedReaction`) so they tick on
|
|
59
|
+
* reanimated-web; the combiner is a pure-read `useDerivedValue`. Effect-free — no
|
|
60
|
+
* React `useEffect`.
|
|
61
|
+
*/
|
|
62
|
+
export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenerating }: UseAmbientWaveOptions): UseAmbientWaveResult {
|
|
63
|
+
// ── Idle breath: subtle 0.04↔0.09 swell (~7s), started once on mount ──
|
|
64
|
+
const idleBreath = useSharedValue(0.04);
|
|
65
|
+
useAnimatedReaction(
|
|
66
|
+
() => true,
|
|
67
|
+
(_current, previous) => {
|
|
68
|
+
if (previous !== null) return; // fire only on the initial run
|
|
69
|
+
idleBreath.value = withRepeat(
|
|
70
|
+
withSequence(
|
|
71
|
+
withTiming(0.09, { duration: 3500, easing: Easing.inOut(Easing.sin) }),
|
|
72
|
+
withTiming(0.04, { duration: 3500, easing: Easing.inOut(Easing.sin) }),
|
|
73
|
+
),
|
|
74
|
+
-1,
|
|
75
|
+
);
|
|
76
|
+
},
|
|
77
|
+
[],
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
// ── STT metering smoothing (floor 0.08, fast attack / slow decay, →0 on stop) ──
|
|
81
|
+
const sttIsRecording = useSTTStore((s) => s.isRecording);
|
|
82
|
+
const sttMetering = useSTTStore((s) => s.metering);
|
|
83
|
+
const stt = useSharedValue(0);
|
|
84
|
+
useAnimatedReaction(
|
|
85
|
+
() => (sttIsRecording ? Math.max(0.08, sttMetering) : -1),
|
|
86
|
+
(target) => {
|
|
87
|
+
if (target < 0) {
|
|
88
|
+
stt.value = withTiming(0, { duration: 300 });
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
const duration = target > stt.value ? 60 : 200;
|
|
92
|
+
stt.value = withTiming(target, { duration, easing: Easing.bezier(0.33, 1, 0.68, 1) });
|
|
93
|
+
},
|
|
94
|
+
[sttIsRecording, sttMetering],
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
// ── Thinking/generating motion (text chat, no voice) — gentle oscillation ──
|
|
98
|
+
const thinkingAmp = useSharedValue(0);
|
|
99
|
+
useAnimatedReaction(
|
|
100
|
+
() => isGenerating ?? false,
|
|
101
|
+
(generating, previous) => {
|
|
102
|
+
if (generating === previous) return;
|
|
103
|
+
if (generating) {
|
|
104
|
+
thinkingAmp.value = withRepeat(
|
|
105
|
+
withSequence(
|
|
106
|
+
withTiming(0.18, { duration: 1200, easing: Easing.inOut(Easing.sin) }),
|
|
107
|
+
withTiming(0.08, { duration: 1200, easing: Easing.inOut(Easing.sin) }),
|
|
108
|
+
),
|
|
109
|
+
-1,
|
|
110
|
+
);
|
|
111
|
+
} else {
|
|
112
|
+
cancelAnimation(thinkingAmp);
|
|
113
|
+
thinkingAmp.value = withTiming(0, { duration: 300 });
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
[isGenerating],
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
// ── Combined amplitude — pure reads only (ticks on web) ──
|
|
120
|
+
const voiceAmplitude = voice?.waveAmplitude;
|
|
121
|
+
const waveAmplitude = useDerivedValue(
|
|
122
|
+
() => Math.max(idleBreath.value, voiceAmplitude ? voiceAmplitude.value : 0, ttsWaveAmplitude.value, stt.value, thinkingAmp.value),
|
|
123
|
+
[voiceAmplitude, ttsWaveAmplitude],
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
// ── Mode → palette state + overlay intensity (plain derived) ──
|
|
127
|
+
return useMemo<UseAmbientWaveResult>(() => {
|
|
128
|
+
if (voice?.isActive && voice.isConnected) {
|
|
129
|
+
return { waveAmplitude, agentState: voice.agentState, intensity: ACTIVE_INTENSITY, mode: 'voice' };
|
|
130
|
+
}
|
|
131
|
+
if (isTTSPlaying) {
|
|
132
|
+
return { waveAmplitude, agentState: 'speaking', intensity: ACTIVE_INTENSITY, mode: 'tts' };
|
|
133
|
+
}
|
|
134
|
+
if (sttIsRecording) {
|
|
135
|
+
return { waveAmplitude, agentState: 'listening', intensity: ACTIVE_INTENSITY, mode: 'stt' };
|
|
136
|
+
}
|
|
137
|
+
if (isGenerating) {
|
|
138
|
+
return { waveAmplitude, agentState: 'thinking', intensity: THINKING_INTENSITY, mode: 'thinking' };
|
|
139
|
+
}
|
|
140
|
+
return { waveAmplitude, agentState: 'idle', intensity: IDLE_INTENSITY, mode: 'idle' };
|
|
141
|
+
}, [waveAmplitude, voice?.isActive, voice?.isConnected, voice?.agentState, isTTSPlaying, sttIsRecording, isGenerating]);
|
|
142
|
+
}
|
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { useState, useEffect, useRef } from 'react';
|
|
12
|
-
import { Room, RoomEvent, Track } from 'livekit-client';
|
|
12
|
+
import { Room, RoomEvent, Track, type RemoteTrack, type RemoteTrackPublication, type RemoteParticipant } from 'livekit-client';
|
|
13
13
|
|
|
14
14
|
/** Compute RMS (root mean square) amplitude from an AnalyserNode. Returns 0-1. */
|
|
15
|
-
function getRMS(analyser: AnalyserNode, buffer:
|
|
15
|
+
function getRMS(analyser: AnalyserNode, buffer: Float32Array<ArrayBuffer>): number {
|
|
16
16
|
analyser.getFloatTimeDomainData(buffer);
|
|
17
17
|
let sum = 0;
|
|
18
18
|
for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i];
|
|
@@ -45,8 +45,8 @@ export function useAudioLevelMonitor(room: Room | null, isConnected: boolean) {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
let localAnalyser: AnalyserNode | null = null;
|
|
48
|
-
let localBuffer: Float32Array | null = null;
|
|
49
|
-
const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array }>();
|
|
48
|
+
let localBuffer: Float32Array<ArrayBuffer> | null = null;
|
|
49
|
+
const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array<ArrayBuffer> }>();
|
|
50
50
|
let rafId = 0;
|
|
51
51
|
let disposed = false;
|
|
52
52
|
|
|
@@ -89,7 +89,7 @@ export function useAudioLevelMonitor(room: Room | null, isConnected: boolean) {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
// Listen for new remote audio tracks
|
|
92
|
-
const onTrackSubscribed = (track:
|
|
92
|
+
const onTrackSubscribed = (track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant) => {
|
|
93
93
|
if (track.kind === Track.Kind.Audio && track.mediaStreamTrack) {
|
|
94
94
|
setupRemoteAnalyser(track.mediaStreamTrack, `${participant.identity}-${publication.trackSid}`);
|
|
95
95
|
}
|
|
@@ -97,7 +97,7 @@ export function useAudioLevelMonitor(room: Room | null, isConnected: boolean) {
|
|
|
97
97
|
room.on(RoomEvent.TrackSubscribed, onTrackSubscribed);
|
|
98
98
|
|
|
99
99
|
// Clean up stale analysers when tracks are unsubscribed
|
|
100
|
-
const onTrackUnsubscribed = (
|
|
100
|
+
const onTrackUnsubscribed = (_track: RemoteTrack, publication: RemoteTrackPublication, participant: RemoteParticipant) => {
|
|
101
101
|
const key = `${participant.identity}-${publication.trackSid}`;
|
|
102
102
|
remoteAnalysers.delete(key);
|
|
103
103
|
};
|
|
@@ -21,7 +21,7 @@ export function useDocumentPicker(): UseDocumentPickerResult {
|
|
|
21
21
|
});
|
|
22
22
|
|
|
23
23
|
if (result.canceled === false && result.assets && result.assets.length > 0) {
|
|
24
|
-
return result.assets.map((asset
|
|
24
|
+
return result.assets.map((asset) => ({
|
|
25
25
|
uri: asset.uri,
|
|
26
26
|
name: asset.name,
|
|
27
27
|
size: asset.size || 0,
|
|
@@ -22,7 +22,7 @@ export function useImagePicker(): ImagePickerResult {
|
|
|
22
22
|
});
|
|
23
23
|
|
|
24
24
|
if (!result.canceled && result.assets.length > 0) {
|
|
25
|
-
return result.assets.map((asset
|
|
25
|
+
return result.assets.map((asset) => ({
|
|
26
26
|
uri: asset.uri,
|
|
27
27
|
name: asset.fileName || `image-${Date.now()}.jpg`,
|
|
28
28
|
size: asset.fileSize || 0,
|
|
@@ -8,21 +8,22 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { useEffect, useRef, useCallback } from 'react';
|
|
11
|
+
import type { AudioPlayer, AudioSource } from 'expo-audio';
|
|
11
12
|
import type { AgentState } from '../types';
|
|
12
13
|
|
|
13
14
|
export type SoundName = 'thinking' | 'toolCall' | 'voiceConnect' | 'voiceDisconnect';
|
|
14
15
|
|
|
15
16
|
export interface SoundSources {
|
|
16
|
-
thinking?:
|
|
17
|
-
toolCall?:
|
|
18
|
-
voiceConnect?:
|
|
19
|
-
voiceDisconnect?:
|
|
17
|
+
thinking?: AudioSource;
|
|
18
|
+
toolCall?: AudioSource;
|
|
19
|
+
voiceConnect?: AudioSource;
|
|
20
|
+
voiceDisconnect?: AudioSource;
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
const SOUND_NAMES: SoundName[] = ['thinking', 'toolCall', 'voiceConnect', 'voiceDisconnect'];
|
|
23
24
|
|
|
24
25
|
export function useSoundEffects(enabled: boolean = false, sounds: SoundSources = {}) {
|
|
25
|
-
const playersRef = useRef<Map<SoundName,
|
|
26
|
+
const playersRef = useRef<Map<SoundName, AudioPlayer>>(new Map());
|
|
26
27
|
const loadedRef = useRef(false);
|
|
27
28
|
|
|
28
29
|
useEffect(() => {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { useState, useCallback, useRef, useEffect } from 'react';
|
|
2
|
+
import { errorMessage } from '../lib/utils';
|
|
2
3
|
import {
|
|
3
4
|
useAudioRecorder,
|
|
4
5
|
useAudioRecorderState,
|
|
5
6
|
RecordingPresets,
|
|
6
|
-
|
|
7
|
+
getRecordingPermissionsAsync,
|
|
7
8
|
setAudioModeAsync,
|
|
8
9
|
} from 'expo-audio';
|
|
9
10
|
import { useOxy } from '@oxyhq/services';
|
|
@@ -92,7 +93,12 @@ export function useSpeechToText(options: UseSTTOptions = {}) {
|
|
|
92
93
|
try {
|
|
93
94
|
setError(null);
|
|
94
95
|
|
|
95
|
-
|
|
96
|
+
// getRecordingPermissionsAsync consults the Permissions API first (and
|
|
97
|
+
// only falls back to a real getUserMedia prompt when undetermined), so an
|
|
98
|
+
// already-granted permission never gets misreported as denied when the
|
|
99
|
+
// device itself is missing/busy — that failure surfaces from
|
|
100
|
+
// prepareToRecordAsync below with an accurate message.
|
|
101
|
+
const permStatus = await getRecordingPermissionsAsync();
|
|
96
102
|
if (!permStatus.granted) {
|
|
97
103
|
setError('Microphone permission required');
|
|
98
104
|
return;
|
|
@@ -108,9 +114,17 @@ export function useSpeechToText(options: UseSTTOptions = {}) {
|
|
|
108
114
|
await audioRecorder.prepareToRecordAsync();
|
|
109
115
|
audioRecorder.record();
|
|
110
116
|
isRecordingRef.current = true;
|
|
111
|
-
} catch (e:
|
|
112
|
-
|
|
113
|
-
|
|
117
|
+
} catch (e: unknown) {
|
|
118
|
+
const name = e instanceof Error ? e.name : '';
|
|
119
|
+
if (name === 'NotFoundError' || name === 'OverconstrainedError') {
|
|
120
|
+
setError('No microphone found — check your input devices');
|
|
121
|
+
} else if (name === 'NotAllowedError') {
|
|
122
|
+
setError('Microphone permission required');
|
|
123
|
+
} else if (name === 'NotReadableError') {
|
|
124
|
+
setError('Microphone is in use by another application');
|
|
125
|
+
} else {
|
|
126
|
+
setError('Failed to start recording');
|
|
127
|
+
}
|
|
114
128
|
setState('idle');
|
|
115
129
|
isRecordingRef.current = false;
|
|
116
130
|
}
|
|
@@ -167,9 +181,9 @@ export function useSpeechToText(options: UseSTTOptions = {}) {
|
|
|
167
181
|
const result = await transcribeResponse.json() as { text: string };
|
|
168
182
|
setState('idle');
|
|
169
183
|
return result.text || null;
|
|
170
|
-
} catch (e:
|
|
184
|
+
} catch (e: unknown) {
|
|
171
185
|
console.error('[STT] Transcription error:', e);
|
|
172
|
-
setError(e
|
|
186
|
+
setError(errorMessage(e, 'Transcription failed'));
|
|
173
187
|
setState('idle');
|
|
174
188
|
return null;
|
|
175
189
|
}
|
package/src/hooks/useTTS.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef } from 'react';
|
|
2
|
+
import type { AudioPlayer, AudioPlaybackStatus } from 'expo-audio';
|
|
2
3
|
import {
|
|
3
4
|
useSharedValue,
|
|
4
5
|
withRepeat,
|
|
@@ -8,6 +9,7 @@ import {
|
|
|
8
9
|
Easing,
|
|
9
10
|
} from 'react-native-reanimated';
|
|
10
11
|
import { useOxy } from '@oxyhq/services';
|
|
12
|
+
import { errorMessage } from '../lib/utils';
|
|
11
13
|
import { create } from 'zustand';
|
|
12
14
|
|
|
13
15
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
@@ -63,7 +65,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
63
65
|
reset,
|
|
64
66
|
} = useTTSStore();
|
|
65
67
|
|
|
66
|
-
const playerRef = useRef<
|
|
68
|
+
const playerRef = useRef<AudioPlayer | null>(null);
|
|
67
69
|
|
|
68
70
|
// Simulated wave amplitude for visualization
|
|
69
71
|
const ttsWaveAmplitude = useSharedValue(0);
|
|
@@ -124,7 +126,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
124
126
|
const player = createAudioPlayer({ uri: audioUrl });
|
|
125
127
|
playerRef.current = player;
|
|
126
128
|
|
|
127
|
-
player.addListener('playbackStatusUpdate', (status:
|
|
129
|
+
player.addListener('playbackStatusUpdate', (status: AudioPlaybackStatus) => {
|
|
128
130
|
if (status.didJustFinish) {
|
|
129
131
|
releasePlayer();
|
|
130
132
|
reset();
|
|
@@ -206,9 +208,9 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
206
208
|
|
|
207
209
|
const data = await response.json();
|
|
208
210
|
playFromUrl(data.audioUrl, messageId);
|
|
209
|
-
} catch (e:
|
|
211
|
+
} catch (e: unknown) {
|
|
210
212
|
console.error('[TTS] Error:', e);
|
|
211
|
-
setError(e
|
|
213
|
+
setError(errorMessage(e, 'Failed to read aloud'));
|
|
212
214
|
}
|
|
213
215
|
}, [activeMessageId, playbackState, getToken, apiUrl, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
|
|
214
216
|
|