local-llm-rn 1.0.0

package/local-llm-rn.podspec

@@ -0,0 +1,60 @@
+Pod::Spec.new do |s|
+  s.name         = "local-llm-rn"
+  s.version      = "0.1.0"
+  s.summary      = "Local LLM inference for React Native"
+  s.homepage     = "https://github.com/hilum-labs/local-llm"
+  s.license      = "MIT"
+  s.author       = "Hilum Labs"
+  s.source       = { :git => "." }
+  s.platforms    = { :ios => "16.0" }
+
+  s.source_files = [
+    "ios/**/*.{h,mm,cpp}",
+    "cpp/src/**/*.{c,cpp,h}",
+    "cpp/ggml/src/**/*.{c,cpp,h,m,metal}",
+    "cpp/include/**/*.h",
+    "cpp/ggml/include/**/*.h",
+    "cpp/common/**/*.{c,cpp,h}",
+    "cpp/mtmd/*.{cpp,h}",
+    "cpp/vendor/**/*.{h,hpp}",
+  ]
+
+  s.exclude_files = [
+    "cpp/ggml/src/ggml-cuda/**",
+    "cpp/ggml/src/ggml-vulkan/**",
+    "cpp/ggml/src/ggml-sycl/**",
+    "cpp/ggml/src/ggml-cann/**",
+    "cpp/ggml/src/ggml-hip/**",
+    "cpp/ggml/src/ggml-kompute/**",
+  ]
+
+  s.pod_target_xcconfig = {
+    "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
+    "GCC_PREPROCESSOR_DEFINITIONS" => [
+      "GGML_METAL=1",
+      "GGML_METAL_EMBED_LIBRARY=1",
+      "GGML_METAL_USE_BF16=1",
+      "GGML_USE_METAL=1",
+      "GGML_BLAS=1",
+      "GGML_BLAS_USE_ACCELERATE=1",
+      "NDEBUG=1",
+      "LLAMA_BUILD_TESTS=OFF",
+      "LLAMA_BUILD_EXAMPLES=OFF",
+      "LLAMA_BUILD_SERVER=OFF",
+    ].join(" "),
+    "HEADER_SEARCH_PATHS" => [
+      "$(PODS_TARGET_SRCROOT)/cpp/include",
+      "$(PODS_TARGET_SRCROOT)/cpp/ggml/include",
+      "$(PODS_TARGET_SRCROOT)/cpp/ggml/src",
+      "$(PODS_TARGET_SRCROOT)/cpp/src",
+      "$(PODS_TARGET_SRCROOT)/cpp/common",
+      "$(PODS_TARGET_SRCROOT)/cpp/mtmd",
+      "$(PODS_TARGET_SRCROOT)/cpp/vendor",
+    ].join(" "),
+    "OTHER_CFLAGS" => "-O3 -DACCELERATE_NEW_LAPACK",
+  }
+
+  s.frameworks = ["Metal", "MetalKit", "Foundation", "Accelerate"]
+
+  install_modules_dependencies(s)
+end

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "local-llm-rn",
+  "version": "1.0.0",
+  "description": "Local LLM inference for React Native (iOS)",
+  "main": "src/index.ts",
+  "react-native": "src/index.ts",
+  "files": [
+    "src/",
+    "ios/",
+    "cpp/",
+    "local-llm-rn.podspec",
+    "!cpp/.gitkeep"
+  ],
+  "scripts": {
+    "prepare": "bash scripts/prepare.sh",
+    "prepack": "bash scripts/prepack.sh"
+  },
+  "peerDependencies": {
+    "react": "*",
+    "react-native": ">=0.76"
+  },
+  "dependencies": {
+    "local-llm": "workspace:*"
+  },
+  "codegenConfig": {
+    "name": "LocalLLMSpec",
+    "type": "modules",
+    "jsSrcsDir": "src"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hilum-labs/local-llm"
+  }
+}

package/src/NativeLocalLLM.ts

@@ -0,0 +1,73 @@
+import type { TurboModule } from 'react-native';
+import { TurboModuleRegistry } from 'react-native';
+
+export interface Spec extends TurboModule {
+  // ── Backend info ────────────────────────────────────────────────────────
+  backendInfo(): string;
+  backendVersion(): string;
+
+  // ── Model lifecycle ─────────────────────────────────────────────────────
+  loadModel(path: string, options: Object): Promise<string>;       // → model ID
+  getModelSize(modelId: string): number;
+  freeModel(modelId: string): void;
+
+  // ── Context lifecycle ───────────────────────────────────────────────────
+  createContext(modelId: string, options: Object): string;          // → context ID
+  getContextSize(contextId: string): number;
+  freeContext(contextId: string): void;
+
+  // ── KV cache ────────────────────────────────────────────────────────────
+  kvCacheClear(contextId: string, fromPos: number): void;
+
+  // ── Tokenization ────────────────────────────────────────────────────────
+  tokenize(modelId: string, text: string, addSpecial: boolean, parseSpecial: boolean): number[];
+  detokenize(modelId: string, tokens: number[]): string;
+  applyChatTemplate(modelId: string, messages: Object[], addAssistant: boolean): string;
+
+  // ── Text inference ──────────────────────────────────────────────────────
+  generate(modelId: string, contextId: string, prompt: string, options: Object): Promise<string>;
+  startStream(modelId: string, contextId: string, prompt: string, options: Object): void;
+  stopStream(contextId: string): void;
+
+  // ── Vision ──────────────────────────────────────────────────────────────
+  loadProjector(modelId: string, path: string, options: Object): string;  // → mtmd context ID
+  supportVision(mtmdId: string): boolean;
+  freeMtmdContext(mtmdId: string): void;
+  generateVision(
+    modelId: string, contextId: string, mtmdId: string,
+    prompt: string, imageBase64s: string[], options: Object,
+  ): Promise<string>;
+  startStreamVision(
+    modelId: string, contextId: string, mtmdId: string,
+    prompt: string, imageBase64s: string[], options: Object,
+  ): void;
+
+  // ── Grammar / Structured output ─────────────────────────────────────────
+  jsonSchemaToGrammar(schemaJson: string): string;
+
+  // ── Embeddings ──────────────────────────────────────────────────────────
+  getEmbeddingDimension(modelId: string): number;
+  createEmbeddingContext(modelId: string, options: Object): string;  // → context ID
+  embed(contextId: string, modelId: string, tokens: number[]): number[];
+  embedBatch(contextId: string, modelId: string, tokenArrays: Object): Object;
+
+  // ── Batch inference ─────────────────────────────────────────────────────
+  startBatch(modelId: string, contextId: string, prompts: string[], options: Object): void;
+
+  // ── Model quantization ──────────────────────────────────────────────────
+  quantize(inputPath: string, outputPath: string, options: Object): void;
+
+  // ── Logging ─────────────────────────────────────────────────────────────
+  setLogLevel(level: number): void;
+  enableLogEvents(enabled: boolean): void;
+
+  // ── Downloads ───────────────────────────────────────────────────────────
+  downloadModel(url: string, destPath: string): void;
+  cancelDownload(url: string): void;
+
+  // ── Device capabilities (RN-only) ──────────────────────────────────────
+  getDeviceCapabilities(): Object;
+  getModelStoragePath(): string;
+}
+
+export default TurboModuleRegistry.getEnforcing<Spec>('LocalLLM');

package/src/device.ts

@@ -0,0 +1,50 @@
+import NativeLocalLLM from './NativeLocalLLM';
+
+export interface DeviceCapabilities {
+  totalRAM: number;        // bytes
+  availableRAM: number;    // bytes (respects iOS jetsam limits)
+  gpuName: string;         // e.g. "Apple A16 GPU"
+  metalFamily: number;     // Apple GPU family (5 = A12+, 7 = A14+, 9 = A17+)
+  metalVersion: number;    // 1, 2, or 3
+  iosVersion: string;      // e.g. "17.2.1"
+  isLowPowerMode: boolean;
+}
+
+export function getDeviceCapabilities(): DeviceCapabilities {
+  return NativeLocalLLM.getDeviceCapabilities() as DeviceCapabilities;
+}
+
+export function canRunModel(modelSizeBytes: number): {
+  canRun: boolean;
+  reason?: string;
+  suggestion?: string;
+} {
+  const caps = getDeviceCapabilities();
+  const availableMB = caps.availableRAM / (1024 * 1024);
+  const modelSizeMB = modelSizeBytes / (1024 * 1024);
+
+  // Model needs ~1.2x its file size in RAM (weights + KV cache + overhead)
+  const estimatedUsageMB = modelSizeMB * 1.2;
+
+  if (estimatedUsageMB > availableMB * 0.8) {
+    return {
+      canRun: false,
+      reason: `Model needs ~${Math.round(estimatedUsageMB)} MB but only ${Math.round(availableMB)} MB available`,
+      suggestion: modelSizeMB > 2000
+        ? 'Try a Q4_K_M quantized variant or a smaller model'
+        : 'Close other apps to free memory',
+    };
+  }
+
+  return { canRun: true };
+}
+
+export function recommendQuantization(): string {
+  const caps = getDeviceCapabilities();
+  const totalGB = caps.totalRAM / (1024 * 1024 * 1024);
+
+  if (totalGB >= 8) return 'Q8_0';    // iPhone 16 Pro (8 GB)
+  if (totalGB >= 6) return 'Q6_K';    // iPhone 14/15 Pro (6 GB)
+  if (totalGB >= 4) return 'Q4_K_M';  // iPhone 11-13, 14/15 base (4 GB)
+  return 'Q3_K_S';                     // iPhone 8/X (3 GB) -- minimal
+}

package/src/download-adapter.ts

@@ -0,0 +1,17 @@
+export interface DownloadAdapter {
+  download(
+    url: string,
+    destPath: string,
+    onProgress?: (downloaded: number, total: number, percent: number) => void,
+  ): Promise<void>;
+
+  cancel?(url: string): void;
+
+  supportsResume?: boolean;
+  resume?(
+    url: string,
+    destPath: string,
+    resumeToken: unknown,
+    onProgress?: (downloaded: number, total: number, percent: number) => void,
+  ): Promise<void>;
+}

package/src/index.ts

@@ -0,0 +1,21 @@
+// Override native loader before anything else imports it
+import { setNativeAddon } from 'local-llm/native';
+import { createReactNativeAddon } from './native-bridge';
+setNativeAddon(createReactNativeAddon());
+
+// Re-export everything from the shared package
+export { LocalAI, Model, InferenceContext, EmbeddingContext, ModelManager, ModelPool, ChatCompletions, Embeddings } from 'local-llm';
+export type {
+  LocalAIOptions,
+  ChatMessage, ContentPart, ComputeMode, GenerateOptions, FlashAttentionMode,
+  KvCacheType, ContextOverflowStrategy, ContextOverflowConfig, EmbeddingPoolingType,
+  ResponseFormat, ChatCompletionTool, ChatCompletionToolChoice,
+  ModelOptions, ContextOptions, EmbeddingContextOptions, QuantizationType, QuantizeOptions,
+  BatchResult, DownloadOptions,
+} from 'local-llm';
+
+// RN-specific exports
+export { getDeviceCapabilities, canRunModel, recommendQuantization } from './device';
+export type { DeviceCapabilities } from './device';
+export { createNativeDownloader } from './rn-downloader';
+export type { DownloadAdapter } from './download-adapter';

package/src/native-bridge.ts

@@ -0,0 +1,142 @@
+import NativeLocalLLM from './NativeLocalLLM';
+import { NativeEventEmitter, NativeModules } from 'react-native';
+import type { NativeAddon, NativeModel, NativeContext, NativeMtmdContext } from 'local-llm/native';
+
+const emitter = new NativeEventEmitter(NativeModules.LocalLLM);
+
+type BrandedId<B extends string> = { __id: string; __brand: B };
+const wrapId = <B extends string>(id: string, brand: B) => ({ __id: id, __brand: brand } as unknown);
+const unwrapId = (obj: any): string => obj.__id;
+
+export function createReactNativeAddon(): NativeAddon {
+  return {
+    backendInfo: () => NativeLocalLLM.backendInfo(),
+    backendVersion: () => NativeLocalLLM.backendVersion(),
+
+    loadModel: (path, options) => {
+      const promise = NativeLocalLLM.loadModel(path, options ?? {});
+      return promise.then((id) => wrapId(id, 'NativeModel')) as any;
+    },
+    loadModelFromBuffer: () => { throw new Error('loadModelFromBuffer is not available on React Native'); },
+    getModelSize: (model) => NativeLocalLLM.getModelSize(unwrapId(model)),
+    freeModel: (model) => NativeLocalLLM.freeModel(unwrapId(model)),
+
+    createContext: (model, options) => {
+      const id = NativeLocalLLM.createContext(unwrapId(model), options ?? {});
+      return wrapId(id, 'NativeContext') as NativeContext;
+    },
+    getContextSize: (ctx) => NativeLocalLLM.getContextSize(unwrapId(ctx)),
+    freeContext: (ctx) => NativeLocalLLM.freeContext(unwrapId(ctx)),
+
+    kvCacheClear: (ctx, fromPos) => NativeLocalLLM.kvCacheClear(unwrapId(ctx), fromPos),
+
+    tokenize: (model, text, addSpecial, parseSpecial) => {
+      const arr = NativeLocalLLM.tokenize(unwrapId(model), text, addSpecial ?? true, parseSpecial ?? false);
+      return new Int32Array(arr);
+    },
+    detokenize: (model, tokens) => {
+      const plain = tokens instanceof Int32Array ? Array.from(tokens) : tokens;
+      return NativeLocalLLM.detokenize(unwrapId(model), plain);
+    },
+    applyChatTemplate: (model, messages, addAssistant) =>
+      NativeLocalLLM.applyChatTemplate(unwrapId(model), messages, addAssistant ?? true),
+
+    inferSync: (model, ctx, prompt, options) =>
+      NativeLocalLLM.generate(unwrapId(model), unwrapId(ctx), prompt, options ?? {}) as any,
+
+    inferStream: (model, ctx, prompt, options, callback) => {
+      const ctxId = unwrapId(ctx);
+      const sub = emitter.addListener('onToken', (event) => {
+        if (event.contextId !== ctxId) return;
+        if (event.error) { callback(new Error(event.error), null); sub.remove(); return; }
+        if (event.done) { callback(null, null); sub.remove(); return; }
+        callback(null, event.token);
+      });
+      NativeLocalLLM.startStream(unwrapId(model), ctxId, prompt, options ?? {});
+    },
+
+    createMtmdContext: (model, projectorPath, options) => {
+      const id = NativeLocalLLM.loadProjector(unwrapId(model), projectorPath, options ?? {});
+      return wrapId(id, 'NativeMtmdContext') as NativeMtmdContext;
+    },
+    supportVision: (ctx) => NativeLocalLLM.supportVision(unwrapId(ctx)),
+    freeMtmdContext: (ctx) => NativeLocalLLM.freeMtmdContext(unwrapId(ctx)),
+
+    inferSyncVision: async (model, ctx, mtmdCtx, prompt, imageBuffers, options) => {
+      const base64s = imageBuffers.map((buf) => buf.toString('base64'));
+      return NativeLocalLLM.generateVision(
+        unwrapId(model), unwrapId(ctx), unwrapId(mtmdCtx), prompt, base64s, options ?? {},
+      );
+    },
+
+    inferStreamVision: (model, ctx, mtmdCtx, prompt, imageBuffers, options, callback) => {
+      const ctxId = unwrapId(ctx);
+      const base64s = imageBuffers.map((buf) => buf.toString('base64'));
+      const sub = emitter.addListener('onToken', (event) => {
+        if (event.contextId !== ctxId) return;
+        if (event.error) { callback(new Error(event.error), null); sub.remove(); return; }
+        if (event.done) { callback(null, null); sub.remove(); return; }
+        callback(null, event.token);
+      });
+      NativeLocalLLM.startStreamVision(
+        unwrapId(model), ctxId, unwrapId(mtmdCtx), prompt, base64s, options ?? {},
+      );
+    },
+
+    jsonSchemaToGrammar: (schemaJson) => NativeLocalLLM.jsonSchemaToGrammar(schemaJson),
+
+    getEmbeddingDimension: (model) => NativeLocalLLM.getEmbeddingDimension(unwrapId(model)),
+    createEmbeddingContext: (model, options) => {
+      const id = NativeLocalLLM.createEmbeddingContext(unwrapId(model), options ?? {});
+      return wrapId(id, 'NativeContext') as NativeContext;
+    },
+    embed: (ctx, model, tokens) => {
+      const plain = tokens instanceof Int32Array ? Array.from(tokens) : tokens;
+      return new Float32Array(NativeLocalLLM.embed(unwrapId(ctx), unwrapId(model), plain));
+    },
+    embedBatch: (ctx, model, tokenArrays) => {
+      const plainArrays = tokenArrays.map((t) => (t instanceof Int32Array ? Array.from(t) : t));
+      const results = NativeLocalLLM.embedBatch(unwrapId(ctx), unwrapId(model), plainArrays) as number[][];
+      return results.map((r) => new Float32Array(r));
+    },
+
+    inferBatch: (model, ctx, prompts, options, callback) => {
+      const ctxId = unwrapId(ctx);
+      let completedCount = 0;
+      const sub = emitter.addListener('onBatchToken', (event) => {
+        if (event.contextId !== ctxId) return;
+        if (event.error) { callback(new Error(event.error), null, event.seqIndex, null); return; }
+        if (event.done) {
+          callback(null, null, event.seqIndex, event.finishReason ?? null);
+          completedCount++;
+          if (completedCount >= prompts.length) sub.remove();
+          return;
+        }
+        callback(null, event.token, event.seqIndex, null);
+      });
+      NativeLocalLLM.startBatch(unwrapId(model), ctxId, prompts, options as any);
+    },
+
+    quantize: (inputPath, outputPath, options, callback) => {
+      const sub = emitter.addListener('onQuantizeComplete', (event) => {
+        sub.remove();
+        callback(event.error ? new Error(event.error) : null);
+      });
+      NativeLocalLLM.quantize(inputPath, outputPath, options);
+    },
+
+    setLogCallback: (() => {
+      let logSub: { remove(): void } | null = null;
+      return (cb: ((level: number, text: string) => void) | null) => {
+        if (logSub) { logSub.remove(); logSub = null; }
+        if (cb) {
+          logSub = emitter.addListener('onLog', (event) => cb(event.level, event.text));
+          NativeLocalLLM.enableLogEvents(true);
+        } else {
+          NativeLocalLLM.enableLogEvents(false);
+        }
+      };
+    })(),
+    setLogLevel: (level) => NativeLocalLLM.setLogLevel(level),
+  };
+}

package/src/rn-downloader.ts

@@ -0,0 +1,37 @@
+import NativeLocalLLM from './NativeLocalLLM';
+import { NativeEventEmitter, NativeModules } from 'react-native';
+import type { DownloadAdapter } from './download-adapter';
+
+const emitter = new NativeEventEmitter(NativeModules.LocalLLM);
+
+export function createNativeDownloader(): DownloadAdapter {
+  return {
+    download(url, destPath, onProgress) {
+      return new Promise((resolve, reject) => {
+        const subs = [
+          emitter.addListener('onDownloadProgress', (e) => {
+            if (e.url !== url) return;
+            onProgress?.(e.downloaded, e.total, e.percent);
+          }),
+          emitter.addListener('onDownloadComplete', (e) => {
+            if (e.url !== url) return;
+            subs.forEach((s) => s.remove());
+            resolve();
+          }),
+          emitter.addListener('onDownloadError', (e) => {
+            if (e.url !== url) return;
+            subs.forEach((s) => s.remove());
+            reject(new Error(e.error));
+          }),
+        ];
+        NativeLocalLLM.downloadModel(url, destPath);
+      });
+    },
+
+    cancel(url) {
+      NativeLocalLLM.cancelDownload(url);
+    },
+
+    supportsResume: false,
+  };
+}