cactus-react-native 0.0.1

package/README.md

@@ -0,0 +1,230 @@
+# Cactus for React Native
+
+A lightweight, high-performance framework for running AI models on mobile devices with React Native.
+
+## Installation
+
+```bash
+# Using npm
+npm install cactus-react-native
+
+# Using yarn
+yarn add cactus-react-native
+
+# For iOS, install pods if not on Expo
+npx pod-install
+```
+
+## Basic Usage
+
+### Initialize a Model
+
+```typescript
+import { initLlama, LlamaContext } from 'cactus-react-native';
+
+// Initialize the model
+const context = await initLlama({
+  model: 'models/llama-2-7b-chat.gguf', // Path to your model
+  n_ctx: 2048,                          // Context size
+  n_batch: 512,                         // Batch size for prompt processing
+  n_threads: 4                          // Number of threads to use
+});
+```
+
+### Text Completion
+
+```typescript
+// Generate text completion
+const result = await context.completion({
+  prompt: "Explain quantum computing in simple terms",
+  temperature: 0.7,
+  top_k: 40,
+  top_p: 0.95,
+  n_predict: 512
+}, (token) => {
+  // Process each token as it's generated
+  console.log(token.token);
+});
+
+// Clean up when done
+await context.release();
+```
+
+### Chat Completion
+
+```typescript
+// Chat messages following OpenAI format
+const messages = [
+  { role: "system", content: "You are a helpful assistant." },
+  { role: "user", content: "What is machine learning?" }
+];
+
+// Generate chat completion
+const result = await context.completion({
+  messages: messages,
+  temperature: 0.7,
+  top_k: 40,
+  top_p: 0.95,
+  n_predict: 512
+}, (token) => {
+  // Process each token
+  console.log(token.token);
+});
+```
+
+## Advanced Features
+
+### JSON Mode with Schema Validation
+
+```typescript
+// Define a JSON schema
+const schema = {
+  type: "object",
+  properties: {
+    name: { type: "string" },
+    age: { type: "number" },
+    hobbies: { 
+      type: "array",
+      items: { type: "string" }
+    }
+  },
+  required: ["name", "age"]
+};
+
+// Generate JSON-structured output
+const result = await context.completion({
+  prompt: "Generate a profile for a fictional person",
+  response_format: {
+    type: "json_schema",
+    json_schema: {
+      schema: schema,
+      strict: true
+    }
+  },
+  temperature: 0.7,
+  n_predict: 512
+});
+
+// The result will be valid JSON according to the schema
+const jsonData = JSON.parse(result.text);
+```
+
+### Working with Embeddings
+
+```typescript
+// Generate embeddings for text
+const embedding = await context.embedding("This is a sample text", {
+  pooling_type: "mean" // Options: "none", "mean", "cls", "last", "rank"
+});
+
+console.log(`Embedding dimensions: ${embedding.embedding.length}`);
+// Use the embedding for similarity comparison, clustering, etc.
+```
+
+### Session Management
+
+```typescript
+// Save the current session state
+const tokenCount = await context.saveSession("session.bin", { tokenSize: 1024 });
+console.log(`Saved session with ${tokenCount} tokens`);
+
+// Load a saved session
+const loadResult = await context.loadSession("session.bin");
+console.log(`Loaded session: ${loadResult.success}`);
+```
+
+### Working with LoRA Adapters
+
+```typescript
+// Apply LoRA adapters to the model
+await context.applyLoraAdapters([
+  { path: "models/lora_adapter.bin", scaled: 0.8 }
+]);
+
+// Get currently loaded adapters
+const loadedAdapters = await context.getLoadedLoraAdapters();
+
+// Remove all LoRA adapters
+await context.removeLoraAdapters();
+```
+
+### Model Benchmarking
+
+```typescript
+// Benchmark the model performance
+const benchResult = await context.bench(
+  32,  // pp: prompt processing tests
+  32,  // tg: token generation tests
+  512, // pl: prompt length
+  5    // nr: number of runs
+);
+
+console.log(`Average token generation speed: ${benchResult.tgAvg} tokens/sec`);
+console.log(`Model size: ${benchResult.modelSize} bytes`);
+```
+
+### Native Logging
+
+```typescript
+import { addNativeLogListener, toggleNativeLog } from 'cactus-react-native';
+
+// Enable native logging
+await toggleNativeLog(true);
+
+// Add a listener for native logs
+const logListener = addNativeLogListener((level, text) => {
+  console.log(`[${level}] ${text}`);
+});
+
+// Remove the listener when no longer needed
+logListener.remove();
+```
+
+## Error Handling
+
+```typescript
+try {
+  const context = await initLlama({
+    model: 'models/non-existent-model.gguf',
+    n_ctx: 2048,
+    n_threads: 4
+  });
+} catch (error) {
+  console.error('Failed to initialize model:', error);
+}
+```
+
+## Best Practices
+
+1. **Model Management**
+   - Store models in the app's document directory
+   - Consider model size when targeting specific devices
+   - Smaller models like SmolLM (135M) work well on most devices
+
+2. **Performance Optimization**
+   - Adjust `n_threads` based on the device's capabilities
+   - Use a smaller `n_ctx` for memory-constrained devices
+   - Consider INT8 or INT4 quantized models for better performance
+
+3. **Battery Efficiency**
+   - Release the model context when not in use
+   - Process inference in smaller batches
+   - Consider background processing for long generations
+
+4. **Memory Management**
+   - Always call `context.release()` when done with a model
+   - Use `releaseAllLlama()` when switching between multiple models
+
+## Example App
+
+For a complete working example, check out the [React Native example app](https://github.com/cactus-compute/cactus/tree/main/examples/react-example) in the repository.
+
+This example demonstrates:
+- Loading and initializing models
+- Building a chat interface
+- Streaming responses
+- Proper resource management
+
+## License
+
+This project is licensed under the MIT License.

package/android/build.gradle

@@ -0,0 +1,104 @@
+buildscript {
+  repositories {
+    google()
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath "com.android.tools.build:gradle:7.2.1"
+  }
+}
+
+def isNewArchitectureEnabled() {
+  return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true"
+}
+
+apply plugin: "com.android.library"
+
+
+def appProject = rootProject.allprojects.find { it.plugins.hasPlugin('com.android.application') }
+
+if (isNewArchitectureEnabled()) {
+  apply plugin: "com.facebook.react"
+}
+
+def getExtOrDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties["Cactus_" + name]
+}
+
+def getExtOrIntegerDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["Cactus_" + name]).toInteger()
+}
+
+def reactNativeArchitectures() {
+  def value = project.getProperties().get("reactNativeArchitectures")
+  def archs = value ? value.split(",") : ["x86_64", "arm64-v8a"]
+  return archs.findAll { it != "armeabi-v7a" && it != "x86" } // Not building for 32-bit architectures
+}
+
+android {
+  ndkVersion getExtOrDefault("ndkVersion")
+  def ndkVersionMajor = ndkVersion.split("\\.")[0].toInteger()
+  if (ndkVersionMajor < 24) {
+    ndkVersion = project.properties["Cactus_ndkversion"]
+  }
+  compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
+
+  defaultConfig {
+    minSdkVersion getExtOrIntegerDefault("minSdkVersion")
+    targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
+    buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+    externalNativeBuild {
+      cmake {
+        abiFilters (*reactNativeArchitectures())
+      }
+    }
+  }
+  
+  buildTypes {
+    release {
+      minifyEnabled false
+    }
+  }
+
+  lintOptions {
+    disable "GradleCompatible"
+  }
+
+  compileOptions {
+    sourceCompatibility JavaVersion.VERSION_1_8
+    targetCompatibility JavaVersion.VERSION_1_8
+  }
+
+  sourceSets {
+    main {
+      if (isNewArchitectureEnabled()) {
+        java.srcDirs += ['src/newarch']
+      } else {
+        java.srcDirs += ['src/oldarch']
+      }
+      jniLibs.srcDirs = ['src/main/jniLibs']
+    }
+  }
+}
+
+repositories {
+  mavenCentral()
+  google()
+}
+
+
+dependencies {
+  // For < 0.71, this will be from the local maven repo
+  // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin
+  //noinspection GradleDynamicVersion
+  implementation "com.facebook.react:react-native:+"
+}
+
+if (isNewArchitectureEnabled()) {
+  react {
+    jsRootDir = file("../src/")
+    libraryName = "Cactus"
+    codegenJavaPackageName = "com.cactus"
+  }
+}

package/android/gradle.properties

@@ -0,0 +1,5 @@
+Cactus_kotlinVersion=1.7.0
+Cactus_minSdkVersion=23
+Cactus_targetSdkVersion=31
+Cactus_compileSdkVersion=31
+Cactus_ndkversion=25.1.8937393

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="com.cactus">
+
+</manifest>

package/android/src/main/CMakeLists.txt

@@ -0,0 +1,104 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(cactus)
+
+set(CMAKE_CXX_STANDARD 17)
+# Use cactus as the C++ source directory
+set(SOURCE_DIR ${CMAKE_SOURCE_DIR}/../../../cactus)
+
+include_directories(${SOURCE_DIR})
+
+set(
+    SOURCE_FILES
+    ${SOURCE_DIR}/ggml.c
+    ${SOURCE_DIR}/ggml-alloc.c
+    ${SOURCE_DIR}/ggml-backend.cpp
+    ${SOURCE_DIR}/ggml-backend-reg.cpp
+    ${SOURCE_DIR}/ggml-cpu.c
+    ${SOURCE_DIR}/ggml-cpu.cpp
+    ${SOURCE_DIR}/ggml-cpu-aarch64.cpp
+    ${SOURCE_DIR}/ggml-cpu-quants.c
+    ${SOURCE_DIR}/ggml-cpu-traits.cpp
+    ${SOURCE_DIR}/ggml-opt.cpp
+    ${SOURCE_DIR}/ggml-threading.cpp
+    ${SOURCE_DIR}/ggml-quants.c
+    ${SOURCE_DIR}/gguf.cpp
+    ${SOURCE_DIR}/log.cpp
+    ${SOURCE_DIR}/llama-impl.cpp
+    ${SOURCE_DIR}/llama-grammar.cpp
+    ${SOURCE_DIR}/llama-sampling.cpp
+    ${SOURCE_DIR}/llama-vocab.cpp
+    ${SOURCE_DIR}/llama-adapter.cpp
+    ${SOURCE_DIR}/llama-chat.cpp
+    ${SOURCE_DIR}/llama-context.cpp
+    ${SOURCE_DIR}/llama-kv-cache.cpp
+    ${SOURCE_DIR}/llama-arch.cpp
+    ${SOURCE_DIR}/llama-batch.cpp
+    ${SOURCE_DIR}/llama-cparams.cpp
+    ${SOURCE_DIR}/llama-hparams.cpp
+    ${SOURCE_DIR}/llama.cpp
+    ${SOURCE_DIR}/llama-model.cpp
+    ${SOURCE_DIR}/llama-model-loader.cpp
+    ${SOURCE_DIR}/llama-mmap.cpp
+    ${SOURCE_DIR}/llama-vocab.cpp
+    ${SOURCE_DIR}/llama-memory.cpp
+    ${SOURCE_DIR}/llama-io.cpp
+    ${SOURCE_DIR}/llama-graph.cpp
+    ${SOURCE_DIR}/sampling.cpp
+    ${SOURCE_DIR}/unicode-data.cpp
+    ${SOURCE_DIR}/unicode.cpp
+    ${SOURCE_DIR}/sgemm.cpp
+    ${SOURCE_DIR}/common.cpp
+    ${SOURCE_DIR}/chat.cpp
+    ${SOURCE_DIR}/json-schema-to-grammar.cpp
+    ${SOURCE_DIR}/json.hpp
+    ${SOURCE_DIR}/amx/amx.cpp
+    ${SOURCE_DIR}/amx/mmq.cpp
+    ${SOURCE_DIR}/minja/minja.hpp
+    ${SOURCE_DIR}/minja/chat-template.hpp
+    ${SOURCE_DIR}/cactus.cpp
+    ${CMAKE_SOURCE_DIR}/jni-utils.h
+    ${CMAKE_SOURCE_DIR}/jni.cpp
+)
+
+find_library(LOG_LIB log)
+
+function(build_library target_name cpu_flags)
+    add_library(
+        ${target_name}
+        SHARED
+        ${SOURCE_FILES}
+    )
+
+    target_link_libraries(${target_name} ${LOG_LIB} android)
+
+    target_compile_options(${target_name} PRIVATE -DLM_GGML_USE_CPU -DLM_GGML_USE_CPU_AARCH64 -pthread ${cpu_flags})
+
+    if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+        target_compile_options(${target_name} PRIVATE -DCACTUS_ANDROID_ENABLE_LOGGING)
+    endif ()
+
+        target_compile_options(${target_name} PRIVATE -O3 -DNDEBUG)
+        target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)
+        target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)
+
+        target_link_options(${target_name} PRIVATE -Wl,--gc-sections)
+        target_link_options(${target_name} PRIVATE -Wl,--exclude-libs,ALL)
+        target_link_options(${target_name} PRIVATE -flto)
+
+endfunction()
+
+# Default target (no specific CPU features)
+build_library("cactus" "")
+
+if (${ANDROID_ABI} STREQUAL "arm64-v8a")
+    build_library("cactus_v8" "-march=armv8-a")
+    build_library("cactus_v8_2" "-march=armv8.2-a")
+    build_library("cactus_v8_2_dotprod" "-march=armv8.2-a+dotprod")
+    build_library("cactus_v8_2_i8mm" "-march=armv8.2-a+i8mm")
+    build_library("cactus_v8_2_dotprod_i8mm" "-march=armv8.2-a+dotprod+i8mm")
+
+elseif (${ANDROID_ABI} STREQUAL "x86_64")
+    build_library("cactus_x86_64" "-march=x86-64" "-mtune=intel" "-msse4.2" "-mpopcnt")
+
+endif ()