cactus-react-native 1.0.2 → 1.2.0

package/src/hooks/useCactusLM.ts

@@ -7,7 +7,8 @@ import type {
   CactusLMCompleteResult,
   CactusLMEmbedParams,
   CactusLMEmbedResult,
-  CactusLMGetModelsParams,
+  CactusLMImageEmbedParams,
+  CactusLMImageEmbedResult,
   CactusLMCompleteParams,
   CactusLMDownloadParams,
 } from '../types/CactusLM';
@@ -84,10 +85,15 @@ export const useCactusLM = ({
         throw new Error(message);
       }
 
+      setError(null);
+
+      if (isDownloaded) {
+        return;
+      }
+
       const thisModel = currentModelRef.current;
       const thisDownloadId = ++currentDownloadIdRef.current;
 
-      setError(null);
       setDownloadProgress(0);
       setIsDownloading(true);
       try {
@@ -135,7 +141,7 @@ export const useCactusLM = ({
         setDownloadProgress(0);
       }
     },
-    [cactusLM, isDownloading]
+    [cactusLM, isDownloading, isDownloaded]
   );
 
   const init = useCallback(async () => {
@@ -163,6 +169,7 @@ export const useCactusLM = ({
       options,
       tools,
       onToken,
+      mode,
     }: CactusLMCompleteParams): Promise<CactusLMCompleteResult> => {
       if (isGenerating) {
         const message = 'CactusLM is already generating';
@@ -182,6 +189,7 @@ export const useCactusLM = ({
             setCompletion((prev) => prev + token);
             onToken?.(token);
           },
+          mode,
         });
       } catch (e) {
         setError(getErrorMessage(e));
@@ -215,6 +223,30 @@ export const useCactusLM = ({
     [cactusLM, isGenerating]
   );
 
+  const imageEmbed = useCallback(
+    async ({
+      imagePath,
+    }: CactusLMImageEmbedParams): Promise<CactusLMImageEmbedResult> => {
+      if (isGenerating) {
+        const message = 'CactusLM is already generating';
+        setError(message);
+        throw new Error(message);
+      }
+
+      setError(null);
+      setIsGenerating(true);
+      try {
+        return await cactusLM.imageEmbed({ imagePath });
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        setIsGenerating(false);
+      }
+    },
+    [cactusLM, isGenerating]
+  );
+
   const stop = useCallback(async () => {
     setError(null);
     try {
@@ -249,20 +281,15 @@ export const useCactusLM = ({
     }
   }, [cactusLM]);
 
-  const getModels = useCallback(
-    async ({ forceRefresh }: CactusLMGetModelsParams = {}): Promise<
-      CactusModel[]
-    > => {
-      setError(null);
-      try {
-        return await cactusLM.getModels({ forceRefresh });
-      } catch (e) {
-        setError(getErrorMessage(e));
-        throw e;
-      }
-    },
-    [cactusLM]
-  );
+  const getModels = useCallback(async (): Promise<CactusModel[]> => {
+    setError(null);
+    try {
+      return await cactusLM.getModels();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    }
+  }, [cactusLM]);
 
   return {
     completion,
@@ -277,6 +304,7 @@ export const useCactusLM = ({
     init,
     complete,
     embed,
+    imageEmbed,
     reset,
     stop,
     destroy,

package/src/hooks/useCactusSTT.ts

@@ -0,0 +1,285 @@
+import { useCallback, useEffect, useState, useRef } from 'react';
+import { CactusSTT } from '../classes/CactusSTT';
+import { CactusFileSystem } from '../native';
+import { getErrorMessage } from '../utils/error';
+import type {
+  CactusSTTParams,
+  CactusSTTTranscribeResult,
+  CactusSTTTranscribeParams,
+  CactusSTTDownloadParams,
+  CactusSTTAudioEmbedParams,
+  CactusSTTAudioEmbedResult,
+} from '../types/CactusSTT';
+import type { CactusModel } from '../types/CactusModel';
+
+export const useCactusSTT = ({
+  model = 'whisper-small',
+  contextSize = 2048,
+}: CactusSTTParams = {}) => {
+  const [cactusSTT, setCactusSTT] = useState(
+    () => new CactusSTT({ model, contextSize })
+  );
+
+  // State
+  const [transcription, setTranscription] = useState('');
+  const [isGenerating, setIsGenerating] = useState(false);
+  const [isInitializing, setIsInitializing] = useState(false);
+  const [isDownloaded, setIsDownloaded] = useState(false);
+  const [isDownloading, setIsDownloading] = useState(false);
+  const [downloadProgress, setDownloadProgress] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+
+  const currentModelRef = useRef(model);
+  const currentDownloadIdRef = useRef(0);
+
+  useEffect(() => {
+    currentModelRef.current = model;
+  }, [model]);
+
+  useEffect(() => {
+    setCactusSTT(new CactusSTT({ model, contextSize }));
+
+    setTranscription('');
+    setIsGenerating(false);
+    setIsInitializing(false);
+    setIsDownloaded(false);
+    setIsDownloading(false);
+    setDownloadProgress(0);
+    setError(null);
+
+    let mounted = true;
+    CactusFileSystem.modelExists(model)
+      .then((exists) => {
+        if (!mounted) {
+          return;
+        }
+        setIsDownloaded(exists);
+      })
+      .catch((e) => {
+        if (!mounted) {
+          return;
+        }
+        setIsDownloaded(false);
+        setError(getErrorMessage(e));
+      });
+
+    return () => {
+      mounted = false;
+    };
+  }, [model, contextSize]);
+
+  useEffect(() => {
+    return () => {
+      cactusSTT.destroy().catch(() => {});
+    };
+  }, [cactusSTT]);
+
+  const download = useCallback(
+    async ({ onProgress }: CactusSTTDownloadParams = {}) => {
+      if (isDownloading) {
+        const message = 'CactusSTT is already downloading';
+        setError(message);
+        throw new Error(message);
+      }
+
+      setError(null);
+
+      if (isDownloaded) {
+        return;
+      }
+
+      const thisModel = currentModelRef.current;
+      const thisDownloadId = ++currentDownloadIdRef.current;
+
+      setDownloadProgress(0);
+      setIsDownloading(true);
+      try {
+        await cactusSTT.download({
+          onProgress: (progress) => {
+            if (
+              currentModelRef.current !== thisModel ||
+              currentDownloadIdRef.current !== thisDownloadId
+            ) {
+              return;
+            }
+
+            setDownloadProgress(progress);
+            onProgress?.(progress);
+          },
+        });
+
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setIsDownloaded(true);
+      } catch (e) {
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        if (
+          currentModelRef.current !== thisModel ||
+          currentDownloadIdRef.current !== thisDownloadId
+        ) {
+          return;
+        }
+
+        setIsDownloading(false);
+        setDownloadProgress(0);
+      }
+    },
+    [cactusSTT, isDownloading, isDownloaded]
+  );
+
+  const init = useCallback(async () => {
+    if (isInitializing) {
+      const message = 'CactusSTT is already initializing';
+      setError(message);
+      throw new Error(message);
+    }
+
+    setError(null);
+    setIsInitializing(true);
+    try {
+      await cactusSTT.init();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    } finally {
+      setIsInitializing(false);
+    }
+  }, [cactusSTT, isInitializing]);
+
+  const transcribe = useCallback(
+    async ({
+      audioFilePath,
+      prompt,
+      options,
+      onToken,
+    }: CactusSTTTranscribeParams): Promise<CactusSTTTranscribeResult> => {
+      if (isGenerating) {
+        const message = 'CactusSTT is already generating';
+        setError(message);
+        throw new Error(message);
+      }
+
+      setError(null);
+      setTranscription('');
+      setIsGenerating(true);
+      try {
+        return await cactusSTT.transcribe({
+          audioFilePath,
+          prompt,
+          options,
+          onToken: (token) => {
+            setTranscription((prev) => prev + token);
+            onToken?.(token);
+          },
+        });
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        setIsGenerating(false);
+      }
+    },
+    [cactusSTT, isGenerating]
+  );
+
+  const audioEmbed = useCallback(
+    async ({
+      audioPath,
+    }: CactusSTTAudioEmbedParams): Promise<CactusSTTAudioEmbedResult> => {
+      if (isGenerating) {
+        const message = 'CactusSTT is already generating';
+        setError(message);
+        throw new Error(message);
+      }
+
+      setError(null);
+      setIsGenerating(true);
+      try {
+        return await cactusSTT.audioEmbed({ audioPath });
+      } catch (e) {
+        setError(getErrorMessage(e));
+        throw e;
+      } finally {
+        setIsGenerating(false);
+      }
+    },
+    [cactusSTT, isGenerating]
+  );
+
+  const stop = useCallback(async () => {
+    setError(null);
+    try {
+      await cactusSTT.stop();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    }
+  }, [cactusSTT]);
+
+  const reset = useCallback(async () => {
+    setError(null);
+    try {
+      await cactusSTT.reset();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    } finally {
+      setTranscription('');
+    }
+  }, [cactusSTT]);
+
+  const destroy = useCallback(async () => {
+    setError(null);
+    try {
+      await cactusSTT.destroy();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    } finally {
+      setTranscription('');
+    }
+  }, [cactusSTT]);
+
+  const getModels = useCallback(async (): Promise<CactusModel[]> => {
+    setError(null);
+    try {
+      return await cactusSTT.getModels();
+    } catch (e) {
+      setError(getErrorMessage(e));
+      throw e;
+    }
+  }, [cactusSTT]);
+
+  return {
+    transcription,
+    isGenerating,
+    isInitializing,
+    isDownloaded,
+    isDownloading,
+    downloadProgress,
+    error,
+
+    download,
+    init,
+    transcribe,
+    audioEmbed,
+    reset,
+    stop,
+    destroy,
+    getModels,
+  };
+};

package/src/index.tsx

@@ -1,8 +1,10 @@
 // Classes
 export { CactusLM } from './classes/CactusLM';
+export { CactusSTT } from './classes/CactusSTT';
 
 // Hooks
 export { useCactusLM } from './hooks/useCactusLM';
+export { useCactusSTT } from './hooks/useCactusSTT';
 
 // Types
 export type { CactusModel } from './types/CactusModel';
@@ -10,14 +12,24 @@ export type {
   CactusLMParams,
   CactusLMDownloadParams,
   Message,
-  Options,
+  CompleteOptions,
   Tool,
   CactusLMCompleteParams,
   CactusLMCompleteResult,
   CactusLMEmbedParams,
   CactusLMEmbedResult,
-  CactusLMGetModelsParams,
+  CactusLMImageEmbedParams,
+  CactusLMImageEmbedResult,
 } from './types/CactusLM';
+export type {
+  CactusSTTParams,
+  CactusSTTDownloadParams,
+  TranscribeOptions,
+  CactusSTTTranscribeParams,
+  CactusSTTTranscribeResult,
+  CactusSTTAudioEmbedParams,
+  CactusSTTAudioEmbedResult,
+} from './types/CactusSTT';
 
 // Config
 export { CactusConfig } from './config/CactusConfig';

package/src/native/Cactus.ts

@@ -1,11 +1,16 @@
 import { NitroModules } from 'react-native-nitro-modules';
 import type { Cactus as CactusSpec } from '../specs/Cactus.nitro';
+import { CactusImage } from './CactusImage';
 import type {
   CactusLMCompleteResult,
   Message,
-  Options,
+  CompleteOptions,
   Tool,
 } from '../types/CactusLM';
+import type {
+  CactusSTTTranscribeResult,
+  TranscribeOptions,
+} from '../types/CactusSTT';
 
 export class Cactus {
   private readonly hybridCactus =
@@ -22,11 +27,30 @@ export class Cactus {
   public async complete(
     messages: Message[],
     responseBufferSize: number,
-    options?: Options,
-    tools?: Tool[],
+    options?: CompleteOptions,
+    tools?: { type: 'function'; function: Tool }[],
     callback?: (token: string, tokenId: number) => void
   ): Promise<CactusLMCompleteResult> {
-    const messagesJson = JSON.stringify(messages);
+    const messagesInternal: Message[] = [];
+    for (const message of messages) {
+      if (!message.images) {
+        messagesInternal.push(message);
+        continue;
+      }
+      const resizedImages: string[] = [];
+      for (const imagePath of message.images) {
+        const resizedImage = await CactusImage.resize(
+          imagePath.replace('file://', ''),
+          128,
+          128,
+          1
+        );
+        resizedImages.push(resizedImage);
+      }
+      messagesInternal.push({ ...message, images: resizedImages });
+    }
+
+    const messagesJson = JSON.stringify(messagesInternal);
     const optionsJson = options
       ? JSON.stringify({
           temperature: options.temperature,
@@ -65,10 +89,76 @@ export class Cactus {
     }
   }
 
+  public async transcribe(
+    audioFilePath: string,
+    prompt: string,
+    responseBufferSize: number,
+    options?: TranscribeOptions,
+    callback?: (token: string, tokenId: number) => void
+  ): Promise<CactusSTTTranscribeResult> {
+    const optionsJson = options
+      ? JSON.stringify({
+          temperature: options.temperature,
+          top_p: options.topP,
+          top_k: options.topK,
+          max_tokens: options.maxTokens,
+          stop_sequences: options.stopSequences,
+        })
+      : undefined;
+
+    const response = await this.hybridCactus.transcribe(
+      audioFilePath.replace('file://', ''),
+      prompt,
+      responseBufferSize,
+      optionsJson,
+      callback
+    );
+
+    try {
+      const parsed = JSON.parse(response);
+
+      return {
+        success: parsed.success,
+        response: parsed.response,
+        timeToFirstTokenMs: parsed.time_to_first_token_ms,
+        totalTimeMs: parsed.total_time_ms,
+        tokensPerSecond: parsed.tokens_per_second,
+        prefillTokens: parsed.prefill_tokens,
+        decodeTokens: parsed.decode_tokens,
+        totalTokens: parsed.total_tokens,
+      };
+    } catch {
+      throw new Error('Unable to parse transcription response');
+    }
+  }
+
   public embed(text: string, embeddingBufferSize: number): Promise<number[]> {
     return this.hybridCactus.embed(text, embeddingBufferSize);
   }
 
+  public async imageEmbed(
+    imagePath: string,
+    embeddingBufferSize: number
+  ): Promise<number[]> {
+    const resizedImage = await CactusImage.resize(
+      imagePath.replace('file://', ''),
+      128,
+      128,
+      1
+    );
+    return this.hybridCactus.imageEmbed(resizedImage, embeddingBufferSize);
+  }
+
+  public audioEmbed(
+    audioPath: string,
+    embeddingBufferSize: number
+  ): Promise<number[]> {
+    return this.hybridCactus.audioEmbed(
+      audioPath.replace('file://', ''),
+      embeddingBufferSize
+    );
+  }
+
   public reset(): Promise<void> {
     return this.hybridCactus.reset();
   }

package/src/native/CactusFileSystem.ts

@@ -35,10 +35,10 @@ export class CactusFileSystem {
 
   public static downloadModel(
     model: string,
+    url: string,
     onProgress?: (progress: number) => void
   ): Promise<void> {
-    const from = `https://vlqqczxwyaodtcdmdmlw.supabase.co/storage/v1/object/public/cactus-models/${model}.zip`;
-    return this.hybridCactusFileSystem.downloadModel(model, from, onProgress);
+    return this.hybridCactusFileSystem.downloadModel(model, url, onProgress);
   }
 
   public static deleteModel(model: string): Promise<void> {

package/src/native/CactusImage.ts

@@ -0,0 +1,20 @@
+import { NitroModules } from 'react-native-nitro-modules';
+import type { CactusImage as CactusImageSpec } from '../specs/CactusImage.nitro';
+
+export class CactusImage {
+  private static readonly hybridCactusImage =
+    NitroModules.createHybridObject<CactusImageSpec>('CactusImage');
+
+  public static base64(path: string): Promise<string> {
+    return this.hybridCactusImage.base64(path);
+  }
+
+  public static resize(
+    path: string,
+    height: number,
+    width: number,
+    quality: number
+  ): Promise<string> {
+    return this.hybridCactusImage.resize(path, height, width, quality);
+  }
+}

package/src/native/index.ts

@@ -2,4 +2,5 @@ export { Cactus } from './Cactus';
 export { CactusCrypto } from './CactusCrypto';
 export { CactusDeviceInfo } from './CactusDeviceInfo';
 export { CactusFileSystem } from './CactusFileSystem';
+export { CactusImage } from './CactusImage';
 export { CactusUtil } from './CactusUtil';

package/src/specs/Cactus.nitro.ts

@@ -13,7 +13,16 @@ export interface Cactus extends HybridObject<{ ios: 'c++'; android: 'c++' }> {
     toolsJson?: string,
     callback?: (token: string, tokenId: number) => void
   ): Promise<string>;
+  transcribe(
+    audioFilePath: string,
+    prompt: string,
+    responseBufferSize: number,
+    optionsJson?: string,
+    callback?: (token: string, tokenId: number) => void
+  ): Promise<string>;
   embed(text: string, embeddingBufferSize: number): Promise<number[]>;
+  imageEmbed(imagePath: string, embeddingBufferSize: number): Promise<number[]>;
+  audioEmbed(audioPath: string, embeddingBufferSize: number): Promise<number[]>;
   reset(): Promise<void>;
   stop(): Promise<void>;
   destroy(): Promise<void>;

package/src/specs/CactusImage.nitro.ts

@@ -0,0 +1,12 @@
+import type { HybridObject } from 'react-native-nitro-modules';
+
+export interface CactusImage
+  extends HybridObject<{ ios: 'swift'; android: 'kotlin' }> {
+  base64(path: string): Promise<string>;
+  resize(
+    path: string,
+    height: number,
+    width: number,
+    quality: number
+  ): Promise<string>;
+}