cactus-react-native 1.4.0 → 1.7.0

package/src/classes/CactusSTT.ts

@@ -6,37 +6,47 @@ import type {
   CactusSTTParams,
   CactusSTTAudioEmbedParams,
   CactusSTTAudioEmbedResult,
+  CactusSTTStreamTranscribeStartOptions,
+  CactusSTTStreamTranscribeProcessParams,
+  CactusSTTStreamTranscribeProcessResult,
+  CactusSTTStreamTranscribeStopResult,
 } from '../types/CactusSTT';
-import { Telemetry } from '../telemetry/Telemetry';
-import { CactusConfig } from '../config/CactusConfig';
-import { Database } from '../api/Database';
-import { getErrorMessage } from '../utils/error';
-import type { CactusSTTModel } from '../types/CactusSTTModel';
+import { getRegistry } from '../modelRegistry';
+import type { CactusModel } from '../types/common';
 
 export class CactusSTT {
   private readonly cactus = new Cactus();
 
   private readonly model: string;
-  private readonly contextSize: number;
+  private readonly options: {
+    quantization: 'int4' | 'int8';
+    pro: boolean;
+  };
 
   private isDownloading = false;
   private isInitialized = false;
   private isGenerating = false;
+  private isStreamTranscribing = false;
 
   private static readonly defaultModel = 'whisper-small';
-  private static readonly defaultContextSize = 2048;
+  private static readonly defaultOptions = {
+    quantization: 'int8' as const,
+    pro: false,
+  };
   private static readonly defaultPrompt =
     '<|startoftranscript|><|en|><|transcribe|><|notimestamps|>';
   private static readonly defaultTranscribeOptions = {
-    maxTokens: 512,
+    maxTokens: 384,
   };
   private static readonly defaultEmbedBufferSize = 4096;
 
-  constructor({ model, contextSize }: CactusSTTParams = {}) {
-    Telemetry.init(CactusConfig.telemetryToken);
-
+  constructor({ model, options }: CactusSTTParams = {}) {
     this.model = model ?? CactusSTT.defaultModel;
-    this.contextSize = contextSize ?? CactusSTT.defaultContextSize;
+    this.options = {
+      quantization:
+        options?.quantization ?? CactusSTT.defaultOptions.quantization,
+      pro: options?.pro ?? CactusSTT.defaultOptions.pro,
+    };
   }
 
   public async download({
@@ -51,17 +61,26 @@ export class CactusSTT {
       throw new Error('CactusSTT is already downloading');
     }
 
-    if (await CactusFileSystem.modelExists(this.model)) {
+    if (await CactusFileSystem.modelExists(this.getModelName())) {
+      console.log('Model already exists', this.getModelName());
       onProgress?.(1.0);
       return;
     }
 
     this.isDownloading = true;
     try {
-      const model = await Database.getSTTModel(this.model);
+      const registry = await getRegistry();
+      const modelConfig =
+        registry[this.model]?.quantization[this.options.quantization];
+      const url = this.options.pro ? modelConfig?.pro?.apple : modelConfig?.url;
+
+      if (!url) {
+        throw new Error(`Model ${this.model} with specified options not found`);
+      }
+
       await CactusFileSystem.downloadModel(
-        this.model,
-        model.downloadUrl,
+        this.getModelName(),
+        url,
         onProgress
       );
     } finally {
@@ -78,20 +97,19 @@ export class CactusSTT {
     if (this.isModelPath(this.model)) {
       modelPath = this.model.replace('file://', '');
     } else {
-      if (!(await CactusFileSystem.modelExists(this.model))) {
-        throw new Error(`Model "${this.model}" is not downloaded`);
+      if (!(await CactusFileSystem.modelExists(this.getModelName()))) {
+        console.log('Model does not exist', this.getModelName());
+        throw new Error(
+          `Model "${this.model}" with options ${JSON.stringify(this.options)} is not downloaded`
+        );
       }
-      modelPath = await CactusFileSystem.getModelPath(this.model);
+      modelPath = await CactusFileSystem.getModelPath(this.getModelName());
     }
 
-    try {
-      await this.cactus.init(modelPath, this.contextSize);
-      Telemetry.logInit(this.model, true);
-      this.isInitialized = true;
-    } catch (error) {
-      Telemetry.logInit(this.model, false, getErrorMessage(error));
-      throw error;
-    }
+    const cacheDir = await CactusFileSystem.getCactusDirectory();
+    await this.cactus.setTelemetryEnvironment(cacheDir);
+    await this.cactus.init(modelPath);
+    this.isInitialized = true;
   }
 
   public async transcribe({
@@ -115,28 +133,52 @@ export class CactusSTT {
 
     this.isGenerating = true;
     try {
-      const result = await this.cactus.transcribe(
+      return await this.cactus.transcribe(
         audio,
         prompt,
         responseBufferSize,
         options,
         onToken
       );
-      Telemetry.logTranscribe(
-        this.model,
-        result.success,
-        result.success ? undefined : result.response,
-        result
-      );
-      return result;
-    } catch (error) {
-      Telemetry.logTranscribe(this.model, false, getErrorMessage(error));
-      throw error;
     } finally {
       this.isGenerating = false;
     }
   }
 
+  public async streamTranscribeStart(
+    options?: CactusSTTStreamTranscribeStartOptions
+  ): Promise<void> {
+    if (this.isStreamTranscribing) {
+      return;
+    }
+
+    await this.init();
+    await this.cactus.streamTranscribeStart(options);
+    this.isStreamTranscribing = true;
+  }
+
+  public async streamTranscribeProcess({
+    audio,
+  }: CactusSTTStreamTranscribeProcessParams): Promise<CactusSTTStreamTranscribeProcessResult> {
+    if (!this.isStreamTranscribing) {
+      throw new Error('CactusSTT stream transcribe is not started');
+    }
+
+    return this.cactus.streamTranscribeProcess(audio);
+  }
+
+  public async streamTranscribeStop(): Promise<CactusSTTStreamTranscribeStopResult> {
+    if (!this.isStreamTranscribing) {
+      throw new Error('CactusSTT stream transcribe is not started');
+    }
+
+    try {
+      return await this.cactus.streamTranscribeStop();
+    } finally {
+      this.isStreamTranscribing = false;
+    }
+  }
+
   public async audioEmbed({
     audioPath,
   }: CactusSTTAudioEmbedParams): Promise<CactusSTTAudioEmbedResult> {
@@ -152,11 +194,7 @@ export class CactusSTT {
         audioPath,
         CactusSTT.defaultEmbedBufferSize
       );
-      Telemetry.logAudioEmbedding(this.model, true);
       return { embedding };
-    } catch (error) {
-      Telemetry.logAudioEmbedding(this.model, false, getErrorMessage(error));
-      throw error;
     } finally {
       this.isGenerating = false;
     }
@@ -177,20 +215,25 @@ export class CactusSTT {
     }
 
     await this.stop();
-    await this.cactus.destroy();
 
+    if (this.isStreamTranscribing) {
+      await this.cactus.streamTranscribeStop().catch(() => {});
+      this.isStreamTranscribing = false;
+    }
+
+    await this.cactus.destroy();
     this.isInitialized = false;
   }
 
-  public async getModels(): Promise<CactusSTTModel[]> {
-    const models = await Database.getSTTModels();
-    for (const model of models) {
-      model.isDownloaded = await CactusFileSystem.modelExists(model.slug);
-    }
-    return models;
+  public async getModels(): Promise<CactusModel[]> {
+    return Object.values(await getRegistry());
   }
 
   private isModelPath(model: string): boolean {
     return model.startsWith('file://') || model.startsWith('/');
   }
+
+  public getModelName(): string {
+    return `${this.model}-${this.options.quantization}${this.options.pro ? '-pro' : ''}`;
+  }
 }

package/src/classes/CactusVAD.ts

@@ -0,0 +1,129 @@
+import { Cactus, CactusFileSystem } from '../native';
+import type {
+  CactusVADParams,
+  CactusVADDownloadParams,
+  CactusVADVadParams,
+  CactusVADResult,
+} from '../types/CactusVAD';
+import { getRegistry } from '../modelRegistry';
+import type { CactusModel } from '../types/common';
+
+export class CactusVAD {
+  private readonly cactus = new Cactus();
+
+  private readonly model: string;
+  private readonly options: {
+    quantization: 'int4' | 'int8';
+    pro: boolean;
+  };
+
+  private isDownloading = false;
+  private isInitialized = false;
+
+  private static readonly defaultModel = 'silero-vad';
+  private static readonly defaultOptions = {
+    quantization: 'int8' as const,
+    pro: false,
+  };
+
+  constructor({ model, options }: CactusVADParams = {}) {
+    this.model = model ?? CactusVAD.defaultModel;
+    this.options = {
+      quantization:
+        options?.quantization ?? CactusVAD.defaultOptions.quantization,
+      pro: options?.pro ?? CactusVAD.defaultOptions.pro,
+    };
+  }
+
+  public async download({
+    onProgress,
+  }: CactusVADDownloadParams = {}): Promise<void> {
+    if (this.isModelPath(this.model)) {
+      onProgress?.(1.0);
+      return;
+    }
+
+    if (this.isDownloading) {
+      throw new Error('CactusVAD is already downloading');
+    }
+
+    if (await CactusFileSystem.modelExists(this.getModelName())) {
+      console.log('Model already exists', this.getModelName());
+      onProgress?.(1.0);
+      return;
+    }
+
+    this.isDownloading = true;
+    try {
+      const registry = await getRegistry();
+      const modelConfig =
+        registry[this.model]?.quantization[this.options.quantization];
+      const url = this.options.pro ? modelConfig?.pro?.apple : modelConfig?.url;
+
+      if (!url) {
+        throw new Error(`Model ${this.model} with specified options not found`);
+      }
+
+      await CactusFileSystem.downloadModel(
+        this.getModelName(),
+        url,
+        onProgress
+      );
+    } finally {
+      this.isDownloading = false;
+    }
+  }
+
+  public async init(): Promise<void> {
+    if (this.isInitialized) {
+      return;
+    }
+
+    let modelPath: string;
+    if (this.isModelPath(this.model)) {
+      modelPath = this.model.replace('file://', '');
+    } else {
+      if (!(await CactusFileSystem.modelExists(this.getModelName()))) {
+        console.log('Model does not exist', this.getModelName());
+        throw new Error(
+          `Model "${this.model}" with options ${JSON.stringify(this.options)} is not downloaded`
+        );
+      }
+      modelPath = await CactusFileSystem.getModelPath(this.getModelName());
+    }
+
+    const cacheDir = await CactusFileSystem.getCactusDirectory();
+    await this.cactus.setTelemetryEnvironment(cacheDir);
+    await this.cactus.init(modelPath);
+    this.isInitialized = true;
+  }
+
+  public async vad({
+    audio,
+    options,
+  }: CactusVADVadParams): Promise<CactusVADResult> {
+    await this.init();
+    return this.cactus.vad(audio, options);
+  }
+
+  public async destroy(): Promise<void> {
+    if (!this.isInitialized) {
+      return;
+    }
+
+    await this.cactus.destroy();
+    this.isInitialized = false;
+  }
+
+  public async getModels(): Promise<CactusModel[]> {
+    return Object.values(await getRegistry());
+  }
+
+  private isModelPath(model: string): boolean {
+    return model.startsWith('file://') || model.startsWith('/');
+  }
+
+  public getModelName(): string {
+    return `${this.model}-${this.options.quantization}${this.options.pro ? '-pro' : ''}`;
+  }
+}

package/src/hooks/useCactusLM.ts

@@ -16,15 +16,19 @@ import type {
   CactusLMImageEmbedResult,
   CactusLMDownloadParams,
 } from '../types/CactusLM';
-import type { CactusModel } from '../types/CactusModel';
+import type { CactusModel } from '../types/common';
 
 export const useCactusLM = ({
-  model = 'qwen3-0.6',
-  contextSize = 2048,
+  model = 'qwen3-0.6b',
   corpusDir = undefined,
+  cacheIndex = false,
+  options: modelOptions = {
+    quantization: undefined,
+    pro: false,
+  },
 }: CactusLMParams = {}) => {
   const [cactusLM, setCactusLM] = useState(
-    () => new CactusLM({ model, contextSize, corpusDir })
+    () => new CactusLM({ model, corpusDir, cacheIndex, options: modelOptions })
   );
 
   // State
@@ -44,7 +48,16 @@ export const useCactusLM = ({
   }, [model]);
 
   useEffect(() => {
-    setCactusLM(new CactusLM({ model, contextSize, corpusDir }));
+    const newInstance = new CactusLM({
+      model,
+      corpusDir,
+      cacheIndex,
+      options: {
+        quantization: modelOptions.quantization,
+        pro: modelOptions.pro,
+      },
+    });
+    setCactusLM(newInstance);
 
     setCompletion('');
     setIsGenerating(false);
@@ -55,7 +68,7 @@ export const useCactusLM = ({
     setError(null);
 
     let mounted = true;
-    CactusFileSystem.modelExists(model)
+    CactusFileSystem.modelExists(newInstance.getModelName())
       .then((exists) => {
         if (!mounted) {
           return;
@@ -73,7 +86,13 @@ export const useCactusLM = ({
     return () => {
       mounted = false;
     };
-  }, [model, contextSize, corpusDir]);
+  }, [
+    model,
+    corpusDir,
+    cacheIndex,
+    modelOptions.quantization,
+    modelOptions.pro,
+  ]);
 
   useEffect(() => {
     return () => {
@@ -173,7 +192,6 @@ export const useCactusLM = ({
       options,
       tools,
       onToken,
-      mode,
     }: CactusLMCompleteParams): Promise<CactusLMCompleteResult> => {
       if (isGenerating) {
         const message = 'CactusLM is already generating';
@@ -193,7 +211,6 @@ export const useCactusLM = ({
             setCompletion((prev) => prev + token);
             onToken?.(token);
           },
-          mode,
         });
       } catch (e) {
         setError(getErrorMessage(e));

package/src/hooks/useCactusSTT.ts

@@ -9,20 +9,31 @@ import type {
   CactusSTTDownloadParams,
   CactusSTTAudioEmbedParams,
   CactusSTTAudioEmbedResult,
+  CactusSTTStreamTranscribeStartOptions,
+  CactusSTTStreamTranscribeProcessParams,
+  CactusSTTStreamTranscribeProcessResult,
+  CactusSTTStreamTranscribeStopResult,
 } from '../types/CactusSTT';
-import type { CactusSTTModel } from '../types/CactusSTTModel';
+import type { CactusModel } from '../types/common';
 
 export const useCactusSTT = ({
   model = 'whisper-small',
-  contextSize = 2048,
+  options: modelOptions = {
+    quantization: undefined,
+    pro: false,
+  },
 }: CactusSTTParams = {}) => {
   const [cactusSTT, setCactusSTT] = useState(
-    () => new CactusSTT({ model, contextSize })
+    () => new CactusSTT({ model, options: modelOptions })
   );
 
   // State
   const [transcription, setTranscription] = useState('');
+  const [streamTranscribeConfirmed, setStreamTranscribeConfirmed] =
+    useState('');
+  const [streamTranscribePending, setStreamTranscribePending] = useState('');
   const [isGenerating, setIsGenerating] = useState(false);
+  const [isStreamTranscribing, setIsStreamTranscribing] = useState(false);
   const [isInitializing, setIsInitializing] = useState(false);
   const [isDownloaded, setIsDownloaded] = useState(false);
   const [isDownloading, setIsDownloading] = useState(false);
@@ -37,10 +48,20 @@ export const useCactusSTT = ({
   }, [model]);
 
   useEffect(() => {
-    setCactusSTT(new CactusSTT({ model, contextSize }));
+    const newInstance = new CactusSTT({
+      model,
+      options: {
+        quantization: modelOptions.quantization,
+        pro: modelOptions.pro,
+      },
+    });
+    setCactusSTT(newInstance);
 
     setTranscription('');
+    setStreamTranscribeConfirmed('');
+    setStreamTranscribePending('');
     setIsGenerating(false);
+    setIsStreamTranscribing(false);
     setIsInitializing(false);
     setIsDownloaded(false);
     setIsDownloading(false);
@@ -48,25 +69,21 @@ export const useCactusSTT = ({
     setError(null);
 
     let mounted = true;
-    CactusFileSystem.modelExists(model)
+    CactusFileSystem.modelExists(newInstance.getModelName())
       .then((exists) => {
-        if (!mounted) {
-          return;
-        }
-        setIsDownloaded(exists);
+        if (mounted) setIsDownloaded(exists);
       })
       .catch((e) => {
-        if (!mounted) {
-          return;
+        if (mounted) {
+          setIsDownloaded(false);
+          setError(getErrorMessage(e));
         }
-        setIsDownloaded(false);
-        setError(getErrorMessage(e));
       });
 
     return () => {
       mounted = false;
     };
-  }, [model, contextSize]);
+  }, [model, modelOptions.quantization, modelOptions.pro]);
 
   useEffect(() => {
     return () => {
@@ -91,49 +108,29 @@ export const useCactusSTT = ({
       const thisModel = currentModelRef.current;
       const thisDownloadId = ++currentDownloadIdRef.current;
 
+      const isCurrent = () =>
+        currentModelRef.current === thisModel &&
+        currentDownloadIdRef.current === thisDownloadId;
+
       setDownloadProgress(0);
       setIsDownloading(true);
       try {
         await cactusSTT.download({
           onProgress: (progress) => {
-            if (
-              currentModelRef.current !== thisModel ||
-              currentDownloadIdRef.current !== thisDownloadId
-            ) {
-              return;
-            }
-
+            if (!isCurrent()) return;
             setDownloadProgress(progress);
             onProgress?.(progress);
           },
         });
 
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setIsDownloaded(true);
       } catch (e) {
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setError(getErrorMessage(e));
         throw e;
       } finally {
-        if (
-          currentModelRef.current !== thisModel ||
-          currentDownloadIdRef.current !== thisDownloadId
-        ) {
-          return;
-        }
-
+        if (!isCurrent()) return;
         setIsDownloading(false);
         setDownloadProgress(0);
       }
@@ -220,6 +217,61 @@ export const useCactusSTT = ({
     [cactusSTT, isGenerating]
   );
 
+  const streamTranscribeStart = useCallback(
+    async (options?: CactusSTTStreamTranscribeStartOptions) => {
+      if (isStreamTranscribing) {
+        return;
+      }
+
+      setError(null);
+      setStreamTranscribeConfirmed('');
+      setStreamTranscribePending('');
+      setIsStreamTranscribing(true);
+      try {
+        await cactusSTT.streamTranscribeStart(options);
+      } catch (e) {
+        setError(getErrorMessage(e));
+        setIsStreamTranscribing(false);
+        throw e;
+      }
+    },
+    [cactusSTT, isStreamTranscribing]
+  );
+
+  const streamTranscribeProcess = useCallback(
+    async ({
+      audio,
+    }: CactusSTTStreamTranscribeProcessParams): Promise<CactusSTTStreamTranscribeProcessResult> => {
+      setError(null);
+      try {
+        const result = await cactusSTT.streamTranscribeProcess({ audio });
+        setStreamTranscribeConfirmed((prev) => prev + result.confirmed);
+        setStreamTranscribePending(result.pending);
+        return result;
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      }
+    },
+    [cactusSTT]
+  );
+
+  const streamTranscribeStop =
+    useCallback(async (): Promise<CactusSTTStreamTranscribeStopResult> => {
+      setError(null);
+      try {
+        const result = await cactusSTT.streamTranscribeStop();
+        setStreamTranscribeConfirmed((prev) => prev + result.confirmed);
+        setStreamTranscribePending('');
+        return result;
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        setIsStreamTranscribing(false);
+      }
+    }, [cactusSTT]);
+
   const stop = useCallback(async () => {
     setError(null);
     try {
@@ -251,10 +303,13 @@ export const useCactusSTT = ({
       throw e;
     } finally {
       setTranscription('');
+      setStreamTranscribeConfirmed('');
+      setStreamTranscribePending('');
+      setIsStreamTranscribing(false);
     }
   }, [cactusSTT]);
 
-  const getModels = useCallback(async (): Promise<CactusSTTModel[]> => {
+  const getModels = useCallback(async (): Promise<CactusModel[]> => {
     setError(null);
     try {
       return await cactusSTT.getModels();
@@ -266,7 +321,10 @@ export const useCactusSTT = ({
 
   return {
     transcription,
+    streamTranscribeConfirmed,
+    streamTranscribePending,
     isGenerating,
+    isStreamTranscribing,
     isInitializing,
     isDownloaded,
     isDownloading,
@@ -277,8 +335,11 @@ export const useCactusSTT = ({
     init,
     transcribe,
     audioEmbed,
-    reset,
+    streamTranscribeStart,
+    streamTranscribeProcess,
+    streamTranscribeStop,
     stop,
+    reset,
     destroy,
     getModels,
   };