@agent-native/core 0.79.26 → 0.80.0

package/dist/client/composer/useVoiceDictation.d.ts

@@ -17,19 +17,26 @@
  * The hook exposes amplitude (0..1) and duration (ms) so the composer can
  * render the Lovable-style live waveform + MM:SS timer.
  */
+import { type VoiceContextPack } from "../../voice/index.js";
 export type VoiceProvider = "auto" | "openai" | "browser" | "google-realtime" | "builder-gemini" | "builder" | "gemini" | "groq";
 export type TranscriptionMode = "mac-native" | "google-realtime" | "batch";
 export interface VoicePrefs {
     provider: VoiceProvider;
     transcriptionMode?: TranscriptionMode;
     instructions?: string;
+    cleanupEnabled?: boolean;
 }
+export interface VoiceCleanupPrefs {
+    enabled?: boolean;
+}
+type VoiceContextPackSource = VoiceContextPack | (() => VoiceContextPack | undefined | Promise<VoiceContextPack | undefined>) | undefined;
 export type VoiceState = "idle" | "starting" | "recording" | "transcribing" | "error";
 export interface UseVoiceDictationOptions {
     onTranscript: (text: string) => void;
     onError?: (message: string) => void;
     /** Called with (accumulatedFinalText, currentInterimText) as speech is recognized in real time. */
     onLiveUpdate?: (finalText: string, interimText: string) => void;
+    contextPack?: VoiceContextPackSource;
 }
 export interface VoiceDictationApi {
     state: VoiceState;
@@ -43,6 +50,8 @@ export interface VoiceDictationApi {
     cancel: () => void;
     dismissError: () => void;
 }
+export declare function providerForTextCleanup(provider: VoiceProvider): VoiceProvider;
 export declare function voiceDictationStartErrorMessage(error: unknown): string;
 export declare function useVoiceDictation(options: UseVoiceDictationOptions): VoiceDictationApi;
+export {};
 //# sourceMappingURL=useVoiceDictation.d.ts.map

package/dist/client/composer/useVoiceDictation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useVoiceDictation.d.ts","sourceRoot":"","sources":["../../../src/client/composer/useVoiceDictation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,MAAM,MAAM,aAAa,GACrB,MAAM,GACN,QAAQ,GACR,SAAS,GACT,iBAAiB,GACjB,gBAAgB,GAChB,SAAS,GACT,QAAQ,GACR,MAAM,CAAC;AAEX,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG,iBAAiB,GAAG,OAAO,CAAC;AAE3E,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,aAAa,CAAC;IACxB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAgDD,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,UAAU,GACV,WAAW,GACX,cAAc,GACd,OAAO,CAAC;AAEZ,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,mGAAmG;IACnG,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;CACjE;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,YAAY,EAAE,MAAM,IAAI,CAAC;CAC1B;AAsFD,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAkCtE;AAeD,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,iBAAiB,CA+tBnB"}
+{"version":3,"file":"useVoiceDictation.d.ts","sourceRoot":"","sources":["../../../src/client/composer/useVoiceDictation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,sBAAsB,CAAC;AAG9B,MAAM,MAAM,aAAa,GACrB,MAAM,GACN,QAAQ,GACR,SAAS,GACT,iBAAiB,GACjB,gBAAgB,GAChB,SAAS,GACT,QAAQ,GACR,MAAM,CAAC;AAEX,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG,iBAAiB,GAAG,OAAO,CAAC;AAE3E,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,aAAa,CAAC;IACxB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,KAAK,sBAAsB,GACvB,gBAAgB,GAChB,CAAC,MAAM,gBAAgB,GAAG,SAAS,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC,GAC5E,SAAS,CAAC;AAoDd,MAAM,MAAM,UAAU,GAClB,MAAM,GACN,UAAU,GACV,WAAW,GACX,cAAc,GACd,OAAO,CAAC;AAEZ,MAAM,WAAW,wBAAwB;IACvC,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,mGAAmG;IACnG,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAChE,WAAW,CAAC,EAAE,sBAAsB,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,UAAU,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,YAAY,EAAE,MAAM,IAAI,CAAC;CAC1B;AAqHD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,aAAa,GAAG,aAAa,CAE7E;AA0DD,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAkCtE;AAeD,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,iBAAiB,CA0xBnB"}

package/dist/client/composer/useVoiceDictation.js

@@ -18,9 +18,12 @@
  * render the Lovable-style live waveform + MM:SS timer.
  */
 import { useCallback, useEffect, useRef, useState } from "react";
+import { applyVoiceContextReplacements, } from "../../voice/index.js";
 import { agentNativePath } from "../api-path.js";
 const PREFS_KEY = "voice-transcription-prefs";
+const CLEANUP_PREFS_KEY = "voice-cleanup-prefs";
 const PREFS_URL = agentNativePath(`/_agent-native/application-state/${PREFS_KEY}`);
+const CLEANUP_PREFS_URL = agentNativePath(`/_agent-native/application-state/${CLEANUP_PREFS_KEY}`);
 const TRANSCRIBE_URL = agentNativePath("/_agent-native/transcribe-voice");
 const GOOGLE_REALTIME_SESSION_URL = agentNativePath("/_agent-native/transcribe-stream/session");
 const GOOGLE_REALTIME_WS_PROTOCOL = "google-realtime.v1";
@@ -55,10 +58,15 @@ function normalizeProviderForMode(mode, provider) {
     return provider === "builder" ? "builder-gemini" : provider;
 }
 async function readVoicePrefs() {
+    const cleanupPrefs = await readVoiceCleanupPrefs();
     try {
         const res = await fetch(PREFS_URL);
-        if (!res.ok)
-            return { provider: await defaultProvider() };
+        if (!res.ok) {
+            return {
+                provider: await defaultProvider(),
+                cleanupEnabled: cleanupPrefs.enabled,
+            };
+        }
         const body = (await res.json());
         const value = body?.value ??
             body;
@@ -74,6 +82,7 @@ async function readVoicePrefs() {
             return {
                 transcriptionMode: mode,
                 provider,
+                cleanupEnabled: cleanupPrefs.enabled,
                 instructions: typeof instructions === "string" ? instructions.trim() : undefined,
             };
         }
@@ -81,7 +90,24 @@ async function readVoicePrefs() {
     catch {
         /* fall through */
     }
-    return { provider: await defaultProvider() };
+    return {
+        provider: await defaultProvider(),
+        cleanupEnabled: cleanupPrefs.enabled,
+    };
+}
+async function readVoiceCleanupPrefs() {
+    try {
+        const res = await fetch(CLEANUP_PREFS_URL);
+        if (!res.ok)
+            return {};
+        const body = (await res.json());
+        const value = body?.value ??
+            body;
+        return { enabled: value?.enabled === true };
+    }
+    catch {
+        return {};
+    }
 }
 function getSpeechRecognitionCtor() {
     if (typeof window === "undefined")
@@ -110,6 +136,52 @@ function pickMimeType() {
     }
     return "audio/webm";
 }
+async function resolveVoiceContextPack(source) {
+    try {
+        const value = typeof source === "function" ? await source() : source;
+        return value && typeof value === "object" ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function appendVoiceContext(form, contextPack) {
+    if (!contextPack)
+        return;
+    try {
+        form.append("voiceContext", JSON.stringify(contextPack));
+    }
+    catch {
+        /* best effort */
+    }
+}
+export function providerForTextCleanup(provider) {
+    return provider === "google-realtime" ? "auto" : provider;
+}
+async function cleanupRecognizedText({ text, provider, instructions, contextPack, }) {
+    if (provider === "browser") {
+        return applyVoiceContextReplacements(text, contextPack);
+    }
+    const form = new FormData();
+    form.append("text", text);
+    form.append("provider", providerForTextCleanup(provider));
+    if (instructions?.trim()) {
+        form.append("instructions", instructions.trim());
+    }
+    appendVoiceContext(form, contextPack);
+    const res = await fetch(TRANSCRIBE_URL, {
+        method: "POST",
+        body: form,
+    });
+    if (!res.ok) {
+        const body = await res
+            .json()
+            .catch(() => ({ error: `HTTP ${res.status}` }));
+        throw new Error(body.error || `Transcript cleanup failed (${res.status})`);
+    }
+    const data = (await res.json());
+    return (data.text ?? "").trim() || text;
+}
 function microphoneBlockedByPermissionsPolicy() {
     if (typeof document === "undefined")
         return false;
@@ -161,13 +233,15 @@ function voiceDictationSpeechErrorMessage(error) {
     return `Speech recognition error: ${error ?? "unknown"}`;
 }
 export function useVoiceDictation(options) {
-    const { onTranscript, onError, onLiveUpdate } = options;
+    const { onTranscript, onError, onLiveUpdate, contextPack } = options;
     const onTranscriptRef = useRef(onTranscript);
     const onErrorRef = useRef(onError);
     const onLiveUpdateRef = useRef(onLiveUpdate);
+    const contextPackRef = useRef(contextPack);
     onTranscriptRef.current = onTranscript;
     onErrorRef.current = onError;
     onLiveUpdateRef.current = onLiveUpdate;
+    contextPackRef.current = contextPack;
     const [state, setState] = useState("idle");
     const [amplitude, setAmplitude] = useState(0);
     const [durationMs, setDurationMs] = useState(0);
@@ -270,6 +344,7 @@ export function useVoiceDictation(options) {
         onErrorRef.current?.(message);
         teardown();
     }, [teardown]);
+    const getVoiceContextPack = useCallback(() => resolveVoiceContextPack(contextPackRef.current), []);
     const startMeter = useCallback((stream) => {
         try {
             const AudioCtor = typeof window !== "undefined"
@@ -356,6 +431,7 @@ export function useVoiceDictation(options) {
                 if (instructions?.trim()) {
                     form.append("instructions", instructions.trim());
                 }
+                appendVoiceContext(form, await getVoiceContextPack());
                 const res = await fetch(TRANSCRIBE_URL, {
                     method: "POST",
                     body: form,
@@ -445,8 +521,8 @@ export function useVoiceDictation(options) {
                 /* best effort — live preview just won't appear */
             }
         }
-    }, [startMeter, startTimer, teardown, failWith]);
-    const startBrowser = useCallback(async () => {
+    }, [startMeter, startTimer, teardown, failWith, getVoiceContextPack]);
+    const startBrowser = useCallback(async (prefs) => {
         const Ctor = getSpeechRecognitionCtor();
         if (!Ctor) {
             throw new Error("Your browser doesn't support speech recognition. Add an OpenAI API key in settings for Whisper transcription.");
@@ -503,15 +579,38 @@ export function useVoiceDictation(options) {
             const wasCancelled = cancelledRef.current;
             cancelledRef.current = false;
             teardown();
-            if (!wasCancelled && text)
-                onTranscriptRef.current?.(text);
-            setState("idle");
+            if (wasCancelled || !text) {
+                setState("idle");
+                return;
+            }
+            void (async () => {
+                let finalText = text;
+                if (prefs.cleanupEnabled) {
+                    setState("transcribing");
+                    try {
+                        finalText = await cleanupRecognizedText({
+                            text,
+                            provider: prefs.provider,
+                            instructions: prefs.instructions,
+                            contextPack: await getVoiceContextPack(),
+                        });
+                    }
+                    catch {
+                        finalText = text;
+                    }
+                }
+                if (!cancelledRef.current && finalText) {
+                    onTranscriptRef.current?.(finalText);
+                }
+                cancelledRef.current = false;
+                setState("idle");
+            })();
         };
         startTimer();
         setState("recording");
         recognition.start();
-    }, [startMeter, startTimer, teardown, failWith]);
-    const startGoogleRealtime = useCallback(async () => {
+    }, [startMeter, startTimer, teardown, failWith, getVoiceContextPack]);
+    const startGoogleRealtime = useCallback(async (prefs) => {
         const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
         if (cancelledRef.current) {
             for (const track of stream.getTracks())
@@ -557,25 +656,48 @@ export function useVoiceDictation(options) {
             if (finalized)
                 return;
             finalized = true;
-            const finalText = realtimeFinalRef.current.trim();
-            const interimText = realtimeInterimRef.current.trim();
-            const text = [finalText, interimText].filter(Boolean).join(" ").trim();
-            teardown();
-            if (cancelledRef.current) {
-                cancelledRef.current = false;
-                setState("idle");
-                return;
-            }
-            if (text) {
-                onTranscriptRef.current?.(text);
+            void (async () => {
+                const finalText = realtimeFinalRef.current.trim();
+                const interimText = realtimeInterimRef.current.trim();
+                const text = [finalText, interimText]
+                    .filter(Boolean)
+                    .join(" ")
+                    .trim();
+                teardown();
+                if (cancelledRef.current) {
+                    cancelledRef.current = false;
+                    setState("idle");
+                    return;
+                }
+                if (text) {
+                    let cleanedText = text;
+                    if (prefs.cleanupEnabled) {
+                        setState("transcribing");
+                        try {
+                            cleanedText = await cleanupRecognizedText({
+                                text,
+                                provider: prefs.provider,
+                                instructions: prefs.instructions,
+                                contextPack: await getVoiceContextPack(),
+                            });
+                        }
+                        catch {
+                            cleanedText = text;
+                        }
+                    }
+                    if (!cancelledRef.current && cleanedText) {
+                        onTranscriptRef.current?.(cleanedText);
+                    }
+                    cancelledRef.current = false;
+                    setState("idle");
+                    return;
+                }
+                if (errorMessage) {
+                    failWith(errorMessage);
+                    return;
+                }
                 setState("idle");
-                return;
-            }
-            if (errorMessage) {
-                failWith(errorMessage);
-                return;
-            }
-            setState("idle");
+            })();
         };
         recorder.ondataavailable = async (event) => {
             if (!event.data || event.data.size === 0)
@@ -703,7 +825,7 @@ export function useVoiceDictation(options) {
                 finish(err?.message ?? "Could not start realtime stream");
             }
         };
-    }, [startMeter, startTimer, teardown, failWith]);
+    }, [startMeter, startTimer, teardown, failWith, getVoiceContextPack]);
     const start = useCallback(async () => {
         if (state === "recording" || state === "starting")
             return;
@@ -740,10 +862,10 @@ export function useVoiceDictation(options) {
                 if (!mediaRecorderSupported) {
                     throw new Error("Your browser doesn't support audio recording, so Google realtime transcription can't start.");
                 }
-                await startGoogleRealtime();
+                await startGoogleRealtime(prefs);
             }
             else {
-                await startBrowser();
+                await startBrowser(prefs);
             }
         }
         catch (err) {