cactus-react-native 1.5.0 → 1.10.0

package/src/hooks/useCactusSTT.ts

@@ -9,23 +9,22 @@ import type {
   CactusSTTDownloadParams,
   CactusSTTAudioEmbedParams,
   CactusSTTAudioEmbedResult,
-  CactusSTTStreamTranscribeInsertParams,
+  CactusSTTStreamTranscribeStartOptions,
   CactusSTTStreamTranscribeProcessParams,
   CactusSTTStreamTranscribeProcessResult,
-  CactusSTTStreamTranscribeFinalizeResult,
+  CactusSTTStreamTranscribeStopResult,
 } from '../types/CactusSTT';
 import type { CactusModel } from '../types/common';
 
 export const useCactusSTT = ({
   model = 'whisper-small',
-  contextSize = 2048,
   options: modelOptions = {
     quantization: undefined,
     pro: false,
   },
 }: CactusSTTParams = {}) => {
   const [cactusSTT, setCactusSTT] = useState(
-    () => new CactusSTT({ model, contextSize, options: modelOptions })
+    () => new CactusSTT({ model, options: modelOptions })
   );
 
   // State
@@ -49,16 +48,14 @@ export const useCactusSTT = ({
   }, [model]);
 
   useEffect(() => {
-    setCactusSTT(
-      new CactusSTT({
-        model,
-        contextSize,
-        options: {
-          quantization: modelOptions.quantization,
-          pro: modelOptions.pro,
-        },
-      })
-    );
+    const newInstance = new CactusSTT({
+      model,
+      options: {
+        quantization: modelOptions.quantization,
+        pro: modelOptions.pro,
+      },
+    });
+    setCactusSTT(newInstance);
 
     setTranscription('');
     setStreamTranscribeConfirmed('');
@@ -72,25 +69,21 @@ export const useCactusSTT = ({
     setError(null);
 
     let mounted = true;
-    CactusFileSystem.modelExists(model)
+    CactusFileSystem.modelExists(newInstance.getModelName())
       .then((exists) => {
-        if (!mounted) {
-          return;
-        }
-        setIsDownloaded(exists);
+        if (mounted) setIsDownloaded(exists);
       })
       .catch((e) => {
-        if (!mounted) {
-          return;
+        if (mounted) {
+          setIsDownloaded(false);
+          setError(getErrorMessage(e));
         }
-        setIsDownloaded(false);
-        setError(getErrorMessage(e));
       });
 
     return () => {
       mounted = false;
     };
-  }, [model, contextSize, modelOptions.quantization, modelOptions.pro]);
+  }, [model, modelOptions.quantization, modelOptions.pro]);
 
   useEffect(() => {
     return () => {
@@ -115,49 +108,29 @@ export const useCactusSTT = ({
       const thisModel = currentModelRef.current;
       const thisDownloadId = ++currentDownloadIdRef.current;
 
+      const isCurrent = () =>
+        currentModelRef.current === thisModel &&
+        currentDownloadIdRef.current === thisDownloadId;
+
       setDownloadProgress(0);
       setIsDownloading(true);
       try {
         await cactusSTT.download({
           onProgress: (progress) => {
-            if (
-              currentModelRef.current !== thisModel ||
-              currentDownloadIdRef.current !== thisDownloadId
-            ) {
-              return;
-            }
-
+            if (!isCurrent()) return;
             setDownloadProgress(progress);
             onProgress?.(progress);
           },
         });
 
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setIsDownloaded(true);
       } catch (e) {
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setError(getErrorMessage(e));
         throw e;
       } finally {
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setIsDownloading(false);
         setDownloadProgress(0);
       }
@@ -244,44 +217,34 @@ export const useCactusSTT = ({
     [cactusSTT, isGenerating]
   );
 
-  const streamTranscribeInit = useCallback(async () => {
-    if (isStreamTranscribing) {
-      return;
-    }
-
-    setError(null);
-    setStreamTranscribeConfirmed('');
-    setStreamTranscribePending('');
-    setIsStreamTranscribing(true);
-    try {
-      await cactusSTT.streamTranscribeInit();
-    } catch (e) {
-      setError(getErrorMessage(e));
-      setIsStreamTranscribing(false);
-      throw e;
-    }
-  }, [cactusSTT, isStreamTranscribing]);
+  const streamTranscribeStart = useCallback(
+    async (options?: CactusSTTStreamTranscribeStartOptions) => {
+      if (isStreamTranscribing) {
+        return;
+      }
 
-  const streamTranscribeInsert = useCallback(
-    async ({ audio }: CactusSTTStreamTranscribeInsertParams): Promise<void> => {
       setError(null);
+      setStreamTranscribeConfirmed('');
+      setStreamTranscribePending('');
+      setIsStreamTranscribing(true);
       try {
-        await cactusSTT.streamTranscribeInsert({ audio });
+        await cactusSTT.streamTranscribeStart(options);
       } catch (e) {
         setError(getErrorMessage(e));
+        setIsStreamTranscribing(false);
         throw e;
       }
     },
-    [cactusSTT]
+    [cactusSTT, isStreamTranscribing]
   );
 
   const streamTranscribeProcess = useCallback(
     async ({
-      options,
-    }: CactusSTTStreamTranscribeProcessParams = {}): Promise<CactusSTTStreamTranscribeProcessResult> => {
+      audio,
+    }: CactusSTTStreamTranscribeProcessParams): Promise<CactusSTTStreamTranscribeProcessResult> => {
       setError(null);
       try {
-        const result = await cactusSTT.streamTranscribeProcess({ options });
+        const result = await cactusSTT.streamTranscribeProcess({ audio });
         setStreamTranscribeConfirmed((prev) => prev + result.confirmed);
         setStreamTranscribePending(result.pending);
         return result;
@@ -293,34 +256,22 @@ export const useCactusSTT = ({
     [cactusSTT]
   );
 
-  const streamTranscribeFinalize =
-    useCallback(async (): Promise<CactusSTTStreamTranscribeFinalizeResult> => {
+  const streamTranscribeStop =
+    useCallback(async (): Promise<CactusSTTStreamTranscribeStopResult> => {
       setError(null);
       try {
-        const result = await cactusSTT.streamTranscribeFinalize();
+        const result = await cactusSTT.streamTranscribeStop();
         setStreamTranscribeConfirmed((prev) => prev + result.confirmed);
         setStreamTranscribePending('');
-        setIsStreamTranscribing(false);
         return result;
       } catch (e) {
         setError(getErrorMessage(e));
         throw e;
+      } finally {
+        setIsStreamTranscribing(false);
       }
     }, [cactusSTT]);
 
-  const streamTranscribeDestroy = useCallback(async (): Promise<void> => {
-    setError(null);
-    try {
-      await cactusSTT.streamTranscribeDestroy();
-    } catch (e) {
-      setError(getErrorMessage(e));
-      throw e;
-    } finally {
-      setIsStreamTranscribing(false);
-      setStreamTranscribePending('');
-    }
-  }, [cactusSTT]);
-
   const stop = useCallback(async () => {
     setError(null);
     try {
@@ -384,13 +335,11 @@ export const useCactusSTT = ({
     init,
     transcribe,
     audioEmbed,
-    streamTranscribeInit,
-    streamTranscribeInsert,
+    streamTranscribeStart,
     streamTranscribeProcess,
-    streamTranscribeFinalize,
-    streamTranscribeDestroy,
-    reset,
+    streamTranscribeStop,
     stop,
+    reset,
     destroy,
     getModels,
   };

package/src/hooks/useCactusVAD.ts

@@ -0,0 +1,215 @@
+import { useCallback, useEffect, useState, useRef } from 'react';
+import { CactusVAD } from '../classes/CactusVAD';
+import { CactusFileSystem } from '../native';
+import { getErrorMessage } from '../utils/error';
+import type {
+  CactusVADParams,
+  CactusVADDownloadParams,
+  CactusVADVadParams,
+  CactusVADResult,
+} from '../types/CactusVAD';
+import type { CactusModel } from '../types/common';
+
+export const useCactusVAD = ({
+  model = 'silero-vad',
+  options: modelOptions = {
+    quantization: undefined,
+    pro: false,
+  },
+}: CactusVADParams = {}) => {
+  const [cactusVAD, setCactusVAD] = useState(
+    () => new CactusVAD({ model, options: modelOptions })
+  );
+
+  const [isInitializing, setIsInitializing] = useState(false);
+  const [isDownloaded, setIsDownloaded] = useState(false);
+  const [isDownloading, setIsDownloading] = useState(false);
+  const [downloadProgress, setDownloadProgress] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+
+  const currentModelRef = useRef(model);
+  const currentDownloadIdRef = useRef(0);
+
+  useEffect(() => {
+    currentModelRef.current = model;
+  }, [model]);
+
+  useEffect(() => {
+    const newInstance = new CactusVAD({
+      model,
+      options: {
+        quantization: modelOptions.quantization,
+        pro: modelOptions.pro,
+      },
+    });
+    setCactusVAD(newInstance);
+
+    setIsInitializing(false);
+    setIsDownloaded(false);
+    setIsDownloading(false);
+    setDownloadProgress(0);
+    setError(null);
+
+    let mounted = true;
+    CactusFileSystem.modelExists(newInstance.getModelName())
+      .then((exists) => {
+        if (!mounted) {
+          return;
+        }
+        setIsDownloaded(exists);
+      })
+      .catch((e) => {
+        if (!mounted) {
+          return;
+        }
+        setIsDownloaded(false);
+        setError(getErrorMessage(e));
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [model, modelOptions.quantization, modelOptions.pro]);
+
+  useEffect(() => {
+    return () => {
+      cactusVAD.destroy().catch(() => {});
+    };
+  }, [cactusVAD]);
+
+  const download = useCallback(
+    async ({ onProgress }: CactusVADDownloadParams = {}) => {
+      if (isDownloading) {
+        const message = 'CactusVAD is already downloading';
+        setError(message);
+        throw new Error(message);
+      }
+
+      setError(null);
+
+      if (isDownloaded) {
+        return;
+      }
+
+      const thisModel = currentModelRef.current;
+      const thisDownloadId = ++currentDownloadIdRef.current;
+
+      setDownloadProgress(0);
+      setIsDownloading(true);
+      try {
+        await cactusVAD.download({
+          onProgress: (progress) => {
+            if (
+              currentModelRef.current !== thisModel ||
+              currentDownloadIdRef.current !== thisDownloadId
+            ) {
+              return;
+            }
+
+            setDownloadProgress(progress);
+            onProgress?.(progress);
+          },
+        });
+
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setIsDownloaded(true);
+      } catch (e) {
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setIsDownloading(false);
+        setDownloadProgress(0);
+      }
+    },
+    [cactusVAD, isDownloading, isDownloaded]
+  );
+
+  const init = useCallback(async () => {
+    if (isInitializing) {
+      const message = 'CactusVAD is already initializing';
+      setError(message);
+      throw new Error(message);
+    }
+
+    setError(null);
+    setIsInitializing(true);
+    try {
+      await cactusVAD.init();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    } finally {
+      setIsInitializing(false);
+    }
+  }, [cactusVAD, isInitializing]);
+
+  const vad = useCallback(
+    async ({
+      audio,
+      options,
+    }: CactusVADVadParams): Promise<CactusVADResult> => {
+      setError(null);
+      try {
+        return await cactusVAD.vad({ audio, options });
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      }
+    },
+    [cactusVAD]
+  );
+
+  const destroy = useCallback(async () => {
+    setError(null);
+    try {
+      await cactusVAD.destroy();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    }
+  }, [cactusVAD]);
+
+  const getModels = useCallback(async (): Promise<CactusModel[]> => {
+    setError(null);
+    try {
+      return await cactusVAD.getModels();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    }
+  }, [cactusVAD]);
+
+  return {
+    isInitializing,
+    isDownloaded,
+    isDownloading,
+    downloadProgress,
+    error,
+
+    download,
+    init,
+    vad,
+    destroy,
+    getModels,
+  };
+};

package/src/index.tsx

@@ -1,21 +1,23 @@
 // Classes
 export { CactusLM } from './classes/CactusLM';
 export { CactusSTT } from './classes/CactusSTT';
+export { CactusVAD } from './classes/CactusVAD';
 export { CactusIndex } from './classes/CactusIndex';
 
 // Hooks
 export { useCactusLM } from './hooks/useCactusLM';
 export { useCactusSTT } from './hooks/useCactusSTT';
+export { useCactusVAD } from './hooks/useCactusVAD';
 export { useCactusIndex } from './hooks/useCactusIndex';
 
 // Types
-export type { CactusModel, ModelOptions } from './types/common';
+export type { CactusModel, CactusModelOptions } from './types/common';
 export type {
   CactusLMParams,
   CactusLMDownloadParams,
-  Message,
-  CompleteOptions,
-  Tool,
+  CactusLMMessage,
+  CactusLMCompleteOptions,
+  CactusLMTool,
   CactusLMCompleteParams,
   CactusLMCompleteResult,
   CactusLMTokenizeParams,
@@ -30,27 +32,34 @@ export type {
 export type {
   CactusSTTParams,
   CactusSTTDownloadParams,
-  TranscribeOptions,
+  CactusSTTTranscribeOptions,
   CactusSTTTranscribeParams,
   CactusSTTTranscribeResult,
   CactusSTTAudioEmbedParams,
   CactusSTTAudioEmbedResult,
-  CactusSTTStreamTranscribeInsertParams,
-  StreamTranscribeProcessOptions,
+  CactusSTTStreamTranscribeStartOptions,
   CactusSTTStreamTranscribeProcessParams,
   CactusSTTStreamTranscribeProcessResult,
-  CactusSTTStreamTranscribeFinalizeResult,
+  CactusSTTStreamTranscribeStopResult,
+  CactusSTTDetectLanguageOptions,
+  CactusSTTDetectLanguageParams,
+  CactusSTTDetectLanguageResult,
 } from './types/CactusSTT';
+export type {
+  CactusVADParams,
+  CactusVADDownloadParams,
+  CactusVADVadParams,
+  CactusVADOptions,
+  CactusVADSegment,
+  CactusVADResult,
+} from './types/CactusVAD';
 export type {
   CactusIndexParams,
   CactusIndexAddParams,
   CactusIndexGetParams,
   CactusIndexGetResult,
-  IndexQueryOptions,
+  CactusIndexQueryOptions,
   CactusIndexQueryParams,
   CactusIndexQueryResult,
   CactusIndexDeleteParams,
 } from './types/CactusIndex';
-
-// Config
-export { CactusConfig } from './config/CactusConfig';

package/src/modelRegistry.ts

@@ -0,0 +1,65 @@
+import type { CactusModel } from './types/common';
+
+const VERSION = 'v1.7';
+
+let registryPromise: Promise<{ [key: string]: CactusModel }> | null = null;
+
+export function getRegistry(): Promise<{ [key: string]: CactusModel }> {
+  return (registryPromise ??= fetchRegistry());
+}
+
+async function fetchRegistry(): Promise<{ [key: string]: CactusModel }> {
+  const response = await fetch(
+    'https://huggingface.co/api/models?author=Cactus-Compute&full=true'
+  ).catch((e) => {
+    registryPromise = null;
+    throw e;
+  });
+  if (!response.ok) {
+    registryPromise = null;
+    throw new Error(`Failed to fetch model registry: ${response.status}`);
+  }
+
+  const models: any[] = await response.json();
+  const registry: { [key: string]: CactusModel } = {};
+
+  for (const { id, siblings = [] } of models) {
+    const weights: string[] = siblings
+      .map((s: any) => s.rfilename)
+      .filter((f: string) => f.startsWith('weights/') && f.endsWith('.zip'));
+
+    if (
+      !weights.some((f) => f.endsWith('-int4.zip')) ||
+      !weights.some((f) => f.endsWith('-int8.zip'))
+    )
+      continue;
+
+    const key = weights
+      .find((f) => f.endsWith('-int4.zip'))!
+      .replace('weights/', '')
+      .replace('-int4.zip', '');
+
+    const base = `https://huggingface.co/${id}/resolve/${VERSION}/weights/${key}`;
+
+    registry[key] = {
+      quantization: {
+        int4: {
+          sizeMb: 0,
+          url: `${base}-int4.zip`,
+          ...(weights.some((f) => f.endsWith('-int4-apple.zip'))
+            ? { pro: { apple: `${base}-int4-apple.zip` } }
+            : {}),
+        },
+        int8: {
+          sizeMb: 0,
+          url: `${base}-int8.zip`,
+          ...(weights.some((f) => f.endsWith('-int8-apple.zip'))
+            ? { pro: { apple: `${base}-int8-apple.zip` } }
+            : {}),
+        },
+      },
+    };
+  }
+
+  return registry;
+}