nastech-app 1.0.0

package/sources/nastech-wire/sessionProtocol.ts

@@ -0,0 +1,134 @@
+import { createId, isCuid } from '@paralleldrive/cuid2';
+import * as z from 'zod';
+
+export const sessionRoleSchema = z.enum(['user', 'agent']);
+export type SessionRole = z.infer<typeof sessionRoleSchema>;
+
+export const sessionTextEventSchema = z.object({
+    t: z.literal('text'),
+    text: z.string(),
+    thinking: z.boolean().optional(),
+});
+
+export const sessionServiceMessageEventSchema = z.object({
+    t: z.literal('service'),
+    text: z.string(),
+});
+
+export const sessionToolCallStartEventSchema = z.object({
+    t: z.literal('tool-call-start'),
+    call: z.string(),
+    name: z.string(),
+    title: z.string(),
+    description: z.string(),
+    args: z.record(z.string(), z.unknown()),
+});
+
+export const sessionToolCallEndEventSchema = z.object({
+    t: z.literal('tool-call-end'),
+    call: z.string(),
+});
+
+export const sessionFileEventSchema = z.object({
+    t: z.literal('file'),
+    ref: z.string(),
+    name: z.string(),
+    size: z.number(),
+    mimeType: z.string().optional(),
+    image: z
+        .object({
+            width: z.number(),
+            height: z.number(),
+            thumbhash: z.string(),
+        })
+        .optional(),
+});
+
+export const sessionTurnStartEventSchema = z.object({
+    t: z.literal('turn-start'),
+});
+
+export const sessionStartEventSchema = z.object({
+    t: z.literal('start'),
+    title: z.string().optional(),
+});
+
+export const sessionTurnEndStatusSchema = z.enum(['completed', 'failed', 'cancelled']);
+export type SessionTurnEndStatus = z.infer<typeof sessionTurnEndStatusSchema>;
+
+export const sessionTurnEndEventSchema = z.object({
+    t: z.literal('turn-end'),
+    status: sessionTurnEndStatusSchema,
+});
+
+export const sessionStopEventSchema = z.object({
+    t: z.literal('stop'),
+});
+
+export const sessionEventSchema = z.discriminatedUnion('t', [
+    sessionTextEventSchema,
+    sessionServiceMessageEventSchema,
+    sessionToolCallStartEventSchema,
+    sessionToolCallEndEventSchema,
+    sessionFileEventSchema,
+    sessionTurnStartEventSchema,
+    sessionStartEventSchema,
+    sessionTurnEndEventSchema,
+    sessionStopEventSchema,
+]);
+
+export type SessionEvent = z.infer<typeof sessionEventSchema>;
+
+export const sessionEnvelopeSchema = z
+    .object({
+        id: z.string(),
+        time: z.number(),
+        role: sessionRoleSchema,
+        turn: z.string().optional(),
+        subagent: z
+            .string()
+            .refine((value) => isCuid(value), {
+                message: 'subagent must be a cuid2 value',
+            })
+            .optional(),
+        claudeUuid: z.string().min(1).optional(),
+        ev: sessionEventSchema,
+    })
+    .superRefine((envelope, ctx) => {
+        if (envelope.ev.t === 'service' && envelope.role !== 'agent') {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: 'service events must use role "agent"',
+                path: ['role'],
+            });
+        }
+        if ((envelope.ev.t === 'start' || envelope.ev.t === 'stop') && envelope.role !== 'agent') {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `${envelope.ev.t} events must use role "agent"`,
+                path: ['role'],
+            });
+        }
+    });
+
+export type SessionEnvelope = z.infer<typeof sessionEnvelopeSchema>;
+
+export type CreateEnvelopeOptions = {
+    id?: string;
+    time?: number;
+    turn?: string;
+    subagent?: string;
+    claudeUuid?: string;
+};
+
+export function createEnvelope(role: SessionRole, ev: SessionEvent, opts: CreateEnvelopeOptions = {}): SessionEnvelope {
+    return sessionEnvelopeSchema.parse({
+        id: opts.id ?? createId(),
+        time: opts.time ?? Date.now(),
+        role,
+        ...(opts.turn ? { turn: opts.turn } : {}),
+        ...(opts.subagent ? { subagent: opts.subagent } : {}),
+        ...(opts.claudeUuid ? { claudeUuid: opts.claudeUuid } : {}),
+        ev,
+    });
+}

package/sources/nastech-wire/voice.ts

@@ -0,0 +1,34 @@
+import * as z from 'zod';
+
+export const VoiceConversationGrantedSchema = z.object({
+    allowed: z.literal(true),
+    conversationToken: z.string(),
+    conversationId: z.string(),
+    agentId: z.string(),
+    elevenUserId: z.string(),
+    usedSeconds: z.number(),
+    limitSeconds: z.number(),
+});
+
+export const VoiceConversationDeniedSchema = z.object({
+    allowed: z.literal(false),
+    reason: z.enum(['voice_hard_limit_reached', 'subscription_required', 'voice_conversation_limit_reached']),
+    usedSeconds: z.number(),
+    limitSeconds: z.number(),
+    agentId: z.string(),
+});
+
+export const VoiceConversationResponseSchema = z.discriminatedUnion('allowed', [
+    VoiceConversationGrantedSchema,
+    VoiceConversationDeniedSchema,
+]);
+export type VoiceConversationResponse = z.infer<typeof VoiceConversationResponseSchema>;
+
+export const VoiceUsageResponseSchema = z.object({
+    usedSeconds: z.number(),
+    limitSeconds: z.number(),
+    conversationCount: z.number(),
+    conversationLimit: z.number(),
+    elevenUserId: z.string(),
+});
+export type VoiceUsageResponse = z.infer<typeof VoiceUsageResponseSchema>;

package/sources/polyfills/screenOrientation.ts

@@ -0,0 +1,33 @@
+/**
+ * Polyfill for `screen.orientation` — react-native-web (and some expo modules)
+ * read `screen.orientation.type` at module-load time. Tauri on certain Linux
+ * setups (X11 fallback, software GL, disabled DMA-BUF) ships a WebKit build
+ * where `screen.orientation` is undefined, which crashes the bundle before
+ * React even mounts.
+ *
+ * Web-only; native platforms have their own AppState/Dimensions stack.
+ * Imported first from index.ts so it runs before any module that touches the API.
+ */
+
+if (typeof window !== 'undefined' && typeof screen !== 'undefined') {
+    const s = screen as unknown as { orientation?: { type: string; angle: number; addEventListener?: Function; removeEventListener?: Function } };
+    if (!s.orientation) {
+        const stub = {
+            type: 'landscape-primary',
+            angle: 0,
+            addEventListener: () => { },
+            removeEventListener: () => { },
+            dispatchEvent: () => false,
+        };
+        try {
+            Object.defineProperty(screen, 'orientation', {
+                value: stub,
+                writable: false,
+                configurable: true,
+            });
+        } catch {
+            // Fallback if defineProperty fails: assign directly
+            (screen as unknown as { orientation: typeof stub }).orientation = stub;
+        }
+    }
+}

package/sources/realtime/RealtimeProvider.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import { ElevenLabsProvider } from '@elevenlabs/react-native';
+import { RealtimeVoiceSession } from './RealtimeVoiceSession';
+import { useVoiceSessionGeneration } from '@/sync/storage';
+
+export const RealtimeProvider = ({ children }: { children: React.ReactNode }) => {
+    // Force ElevenLabsProvider to remount between sessions. The native SDK uses
+    // LiveKit, whose Room instance can't be reused after disconnect — second
+    // startSession silently fails. Children sit OUTSIDE the provider so the app
+    // tree isn't torn down on remount.
+    const generation = useVoiceSessionGeneration();
+    return (
+        <>
+            <ElevenLabsProvider key={generation}>
+                <RealtimeVoiceSession />
+            </ElevenLabsProvider>
+            {children}
+        </>
+    );
+};

package/sources/realtime/RealtimeProvider.web.tsx

@@ -0,0 +1,15 @@
+import React from 'react';
+import { RealtimeVoiceSession } from './RealtimeVoiceSession';
+import { useVoiceSessionGeneration } from '@/sync/storage';
+
+export const RealtimeProvider = ({ children }: { children: React.ReactNode }) => {
+    // Web SDK (@elevenlabs/react) uses a plain WebSocket — no LiveKit Room to
+    // go stale — so this re-key is mostly defensive. Kept symmetric with native.
+    const generation = useVoiceSessionGeneration();
+    return (
+        <>
+            <RealtimeVoiceSession key={generation} />
+            {children}
+        </>
+    );
+};

package/sources/realtime/RealtimeSession.ts

@@ -0,0 +1,200 @@
+import type { VoiceSession } from './types';
+import { fetchVoiceCredentials } from '@/sync/apiVoice';
+import { sync } from '@/sync/sync';
+import { Modal } from '@/modal';
+import { t } from '@/text';
+import { requestMicrophonePermission, showMicrophonePermissionDeniedAlert } from '@/utils/microphonePermissions';
+import { storage } from '@/sync/storage';
+import {
+    getVoiceMessageCount,
+    getVoiceOnboardingPromptLoadCount,
+    getVoiceSoftPaywallShownCount,
+    incrementVoiceOnboardingPromptLoadCount,
+    incrementVoiceSoftPaywallShown,
+} from '@/sync/persistence';
+import { buildVoiceFirstMessage, buildVoiceSystemPrompt } from './voiceSystemPrompt';
+import { getVoiceUpsellVariant } from './voiceExperiment';
+
+let voiceSession: VoiceSession | null = null;
+let voiceSessionStarted: boolean = false;
+let currentSessionId: string | null = null;
+let currentVoiceConversationId: string | null = null;
+let currentVoiceSessionStartedAt: number | null = null;
+
+/**
+ * Start a voice session. Returns the ElevenLabs conversation ID if started, null otherwise.
+ */
+export async function startRealtimeSession(sessionId: string, initialContext?: string): Promise<string | null> {
+    currentVoiceConversationId = null;
+    currentVoiceSessionStartedAt = null;
+
+    if (!voiceSession) {
+        console.warn('No voice session registered');
+        return null;
+    }
+
+    // Show connecting state immediately so the user sees feedback
+    storage.getState().setRealtimeStatus('connecting');
+
+    // Request microphone permission before starting voice session
+    // Critical for iOS/Android - first session will fail without this
+    const permissionResult = await requestMicrophonePermission();
+    if (!permissionResult.granted) {
+        storage.getState().setRealtimeStatus('disconnected');
+        showMicrophonePermissionDeniedAlert(permissionResult.canAskAgain);
+        return null;
+    }
+
+    try {
+        // Bypass NasTech server token — only when user has their own custom agent
+        const { voiceBypassToken, voiceCustomAgentId } = storage.getState().settings;
+        if (voiceBypassToken && voiceCustomAgentId) {
+            console.log('[Voice] Bypassing token, custom agent ID:', voiceCustomAgentId);
+            currentSessionId = sessionId;
+            const conversationId = await voiceSession.startSession({
+                sessionId,
+                initialContext,
+                agentId: voiceCustomAgentId,
+            });
+            currentVoiceConversationId = conversationId;
+            currentVoiceSessionStartedAt = Date.now();
+            voiceSessionStarted = true;
+            return conversationId;
+        }
+
+        const response = await fetchVoiceCredentials(sessionId);
+        console.log('[Voice] fetchVoiceCredentials response:', response);
+
+        if (!response.allowed) {
+            storage.getState().setRealtimeStatus('disconnected');
+
+            if (response.reason === 'voice_conversation_limit_reached') {
+                Modal.alert(
+                    t('errors.voiceLimitReachedTitle'),
+                    t('errors.voiceConversationLimitReached'),
+                );
+                return null;
+            }
+
+            // Server hard-declined — must pay to continue
+            console.log('[Voice] Not allowed (reason: %s), presenting must-pay paywall...', response.reason);
+            const result = await sync.presentPaywall('voice_must_pay');
+            console.log('[Voice] Must-pay paywall result:', result);
+            if (result.purchased) {
+                return startRealtimeSession(sessionId, initialContext);
+            }
+            return null;
+        }
+
+        const hasPro = storage.getState().purchases.entitlements['pro'] ?? false;
+        const { voiceUpsellOverride, devModeEnabled } = storage.getState().localSettings;
+        const voiceUpsellVariant = getVoiceUpsellVariant({
+            override: voiceUpsellOverride,
+            overrideEnabled: __DEV__ || devModeEnabled,
+        });
+
+        if (
+            !hasPro &&
+            voiceUpsellVariant === 'show-paywall-before-first-voice-chat' &&
+            getVoiceSoftPaywallShownCount() < 1
+        ) {
+            console.log('[Voice] First voice attempt on free tier, showing soft paywall...');
+            incrementVoiceSoftPaywallShown();
+            const result = await sync.presentPaywall('voice_trial_eligible');
+            console.log('[Voice] Soft paywall result:', result);
+            // Dismissed or error — continue anyway, they can still use free tier.
+        }
+
+        currentSessionId = sessionId;
+        const onboardingPromptLoadCount = getVoiceOnboardingPromptLoadCount();
+        const voiceMessageCount = getVoiceMessageCount();
+        const systemPrompt = buildVoiceSystemPrompt({
+            initialContext,
+            onboardingPromptLoadCount,
+            voiceMessageCount,
+            includePaidVoiceOnboarding: !hasPro && voiceUpsellVariant === 'voice-onboarding-and-upsell',
+        });
+        const firstMessage = buildVoiceFirstMessage({
+            hasPro,
+            onboardingPromptLoadCount,
+            includePaidVoiceOnboarding: voiceUpsellVariant === 'voice-onboarding-and-upsell',
+        });
+
+        const startedConversationId = await voiceSession.startSession({
+            sessionId,
+            initialContext,
+            systemPrompt,
+            firstMessage,
+            conversationToken: response.conversationToken,
+            agentId: response.agentId,
+            userId: response.elevenUserId,
+        });
+        if (!hasPro && voiceUpsellVariant === 'voice-onboarding-and-upsell') {
+            incrementVoiceOnboardingPromptLoadCount();
+        }
+        currentVoiceConversationId = response.conversationId ?? startedConversationId;
+        currentVoiceSessionStartedAt = Date.now();
+        voiceSessionStarted = true;
+        return currentVoiceConversationId;
+    } catch (error) {
+        console.error('Failed to start realtime session:', error);
+        storage.getState().setRealtimeStatus('disconnected');
+        currentSessionId = null;
+        currentVoiceConversationId = null;
+        currentVoiceSessionStartedAt = null;
+        voiceSessionStarted = false;
+        Modal.alert(t('common.error'), t('errors.voiceServiceUnavailable'));
+        return null;
+    }
+}
+
+export async function stopRealtimeSession() {
+    if (!voiceSession) {
+        return;
+    }
+
+    try {
+        await voiceSession.endSession();
+    } catch (error) {
+        console.error('Failed to stop realtime session:', error);
+    } finally {
+        currentSessionId = null;
+        currentVoiceConversationId = null;
+        currentVoiceSessionStartedAt = null;
+        voiceSessionStarted = false;
+    }
+}
+
+export function registerVoiceSession(session: VoiceSession) {
+    if (voiceSession) {
+        console.warn('Voice session already registered, replacing with new one');
+    }
+    voiceSession = session;
+}
+
+export function isVoiceSessionStarted(): boolean {
+    return voiceSessionStarted;
+}
+
+export function getVoiceSession(): VoiceSession | null {
+    return voiceSession;
+}
+
+export function getCurrentRealtimeSessionId(): string | null {
+    return currentSessionId;
+}
+
+export function getCurrentVoiceConversationId(): string | null {
+    return currentVoiceConversationId;
+}
+
+export function getCurrentVoiceSessionDurationSeconds(): number | undefined {
+    if (currentVoiceSessionStartedAt === null) {
+        return undefined;
+    }
+    return Math.max(0, Math.round((Date.now() - currentVoiceSessionStartedAt) / 1000));
+}
+
+export function setCurrentRealtimeSessionId(sessionId: string) {
+    currentSessionId = sessionId;
+}

package/sources/realtime/RealtimeVoiceSession.tsx

@@ -0,0 +1,211 @@
+import React, { useEffect, useRef } from 'react';
+import { useConversation } from '@elevenlabs/react-native';
+import { registerVoiceSession } from './RealtimeSession';
+import { storage } from '@/sync/storage';
+import { realtimeClientTools } from './realtimeClientTools';
+import { getElevenLabsCodeFromPreference } from '@/constants/Languages';
+import type { VoiceSession, VoiceSessionConfig } from './types';
+
+// Static reference to the conversation hook instance
+let conversationInstance: ReturnType<typeof useConversation> | null = null;
+
+// VAD state for user speech detection
+const VAD_THRESHOLD = 0.5;
+const VAD_SILENCE_MS = 300;
+let vadSilenceTimer: ReturnType<typeof setTimeout> | null = null;
+let agentIsSpeaking = false;
+
+// Global voice session implementation
+class RealtimeVoiceSessionImpl implements VoiceSession {
+    
+    async startSession(config: VoiceSessionConfig): Promise<string | null> {
+        if (!conversationInstance) {
+            console.warn('Realtime voice session not initialized');
+            throw new Error('Realtime voice session not initialized');
+        }
+
+        try {
+            storage.getState().setRealtimeStatus('connecting');
+            
+            // Get user's preferred language for voice assistant
+            const userLanguagePreference = storage.getState().settings.voiceAssistantLanguage;
+            const elevenLabsLanguage = getElevenLabsCodeFromPreference(userLanguagePreference);
+            
+            if (!config.conversationToken && !config.agentId) {
+                throw new Error('No conversationToken or agentId provided');
+            }
+
+            const sessionConfig: any = {
+                // conversationToken (WebRTC JWT from server) or agentId (bypass mode)
+                ...(config.conversationToken
+                    ? { conversationToken: config.conversationToken }
+                    : { agentId: config.agentId }),
+                userId: config.userId,
+                dynamicVariables: {
+                    sessionId: config.sessionId,
+                    initialConversationContext: config.initialContext || ''
+                },
+                overrides: {
+                    agent: {
+                        ...(config.systemPrompt ? { prompt: { prompt: config.systemPrompt } } : {}),
+                        ...(config.firstMessage ? { firstMessage: config.firstMessage } : {}),
+                        language: elevenLabsLanguage
+                    }
+                },
+            };
+            
+            await conversationInstance.startSession(sessionConfig);
+            return conversationInstance.getId?.() ?? null;
+        } catch (error) {
+            console.error('Failed to start realtime session:', error);
+            storage.getState().setRealtimeStatus('error');
+            throw error;
+        }
+    }
+
+    async endSession(): Promise<void> {
+        if (!conversationInstance) {
+            storage.getState().setRealtimeStatus('disconnected');
+            return;
+        }
+
+        try {
+            await conversationInstance.endSession();
+        } catch (error) {
+            console.error('Failed to end realtime session:', error);
+        } finally {
+            storage.getState().setRealtimeStatus('disconnected');
+        }
+    }
+
+    sendTextMessage(message: string): void {
+        if (!conversationInstance) {
+            console.warn('Realtime voice session not initialized');
+            return;
+        }
+
+        try {
+            conversationInstance.sendUserMessage(message);
+        } catch (error) {
+            console.error('Failed to send text message:', error);
+        }
+    }
+
+    sendContextualUpdate(update: string): void {
+        if (!conversationInstance) {
+            console.warn('Realtime voice session not initialized');
+            return;
+        }
+
+        try {
+            conversationInstance.sendContextualUpdate(update);
+        } catch (error) {
+            console.error('Failed to send contextual update:', error);
+        }
+    }
+}
+
+export const RealtimeVoiceSession: React.FC = () => {
+    const conversation = useConversation({
+        clientTools: realtimeClientTools,
+        onConnect: (data) => {
+            console.log('Realtime session connected:', data);
+            storage.getState().setRealtimeStatus('connected');
+            storage.getState().setRealtimeMode('idle');
+        },
+        onDisconnect: () => {
+            console.log('Realtime session disconnected');
+            // Bump generation only when an active session ends — skipping the
+            // initial 'disconnected' state avoids remounting on cold launch
+            // (which previously caused a phantom keyboard).
+            const prev = storage.getState().realtimeStatus;
+            storage.getState().setRealtimeStatus('disconnected');
+            storage.getState().setRealtimeMode('idle', true);
+            storage.getState().clearRealtimeModeDebounce();
+            if (prev === 'connected' || prev === 'connecting') {
+                storage.getState().incrementVoiceSessionGeneration();
+            }
+        },
+        onMessage: (data) => {
+            console.log('Realtime message:', data);
+        },
+        onError: (error) => {
+            // Log but don't block app - voice features will be unavailable
+            // This prevents initialization errors from showing "Terminals error" on startup
+            console.warn('Realtime voice not available:', error);
+            // Don't set error status during initialization - just set disconnected
+            // This allows the app to continue working without voice features.
+            // Don't bump generation here — onError can fire on transient/recoverable
+            // errors (LiveKit retries internally). onDisconnect is where we know
+            // the session is truly dead and a fresh provider is required.
+            storage.getState().setRealtimeStatus('disconnected');
+            storage.getState().setRealtimeMode('idle', true); // immediate mode change
+        },
+        onStatusChange: (data) => {
+            console.log('Realtime status change:', data);
+        },
+        onModeChange: (data) => {
+            console.log('Realtime mode change:', data);
+
+            const mode = data.mode as string;
+            agentIsSpeaking = mode === 'speaking';
+
+            // Use centralized debounce logic from storage
+            if (agentIsSpeaking) {
+                storage.getState().setRealtimeMode('agent-speaking');
+            } else {
+                // Agent stopped speaking — defer to VAD for user-speaking, otherwise idle
+                storage.getState().setRealtimeMode('idle');
+            }
+        },
+        onVadScore: (data) => {
+            const { vadScore } = data;
+            if (agentIsSpeaking) return; // Agent speaking takes priority
+
+            if (vadScore > VAD_THRESHOLD) {
+                if (vadSilenceTimer) {
+                    clearTimeout(vadSilenceTimer);
+                    vadSilenceTimer = null;
+                }
+                storage.getState().setRealtimeMode('user-speaking', true);
+            } else {
+                if (!vadSilenceTimer) {
+                    vadSilenceTimer = setTimeout(() => {
+                        vadSilenceTimer = null;
+                        if (!agentIsSpeaking) {
+                            storage.getState().setRealtimeMode('idle');
+                        }
+                    }, VAD_SILENCE_MS);
+                }
+            }
+        },
+        onDebug: (message) => {
+            console.debug('Realtime debug:', message);
+        }
+    });
+
+    const hasRegistered = useRef(false);
+
+    useEffect(() => {
+        // Store the conversation instance globally
+        conversationInstance = conversation;
+
+        // Register the voice session once
+        if (!hasRegistered.current) {
+            try {
+                registerVoiceSession(new RealtimeVoiceSessionImpl());
+                hasRegistered.current = true;
+            } catch (error) {
+                console.error('Failed to register voice session:', error);
+            }
+        }
+
+        return () => {
+            // Clean up on unmount
+            conversationInstance = null;
+        };
+    }, [conversation]);
+
+    // This component doesn't render anything visible
+    return null;
+};