@inferrlm/react-native-mlx 0.4.0 → 0.4.2-alpha.0

package/MLXReactNative.podspec

@@ -24,7 +24,7 @@ Pod::Spec.new do |s|
 
   spm_dependency(s,
     url: "https://github.com/ml-explore/mlx-swift-lm.git",
-    requirement: {kind: "upToNextMinorVersion", minimumVersion: "2.30.3"},
+    requirement: {kind: "branch", branch: "main"},
     products: ["MLXLLM", "MLXLMCommon"]
   )
 

package/ios/Sources/HybridLLM.swift

@@ -66,6 +66,34 @@ class HybridLLM: HybridLLMSpec {
                      allocatedMB, cacheMB, peakMB)
     }
 
+    private func parseEosTokenIds(from modelDir: URL) -> Set<Int> {
+        let configURL = modelDir.appendingPathComponent("config.json")
+        guard let data = try? Data(contentsOf: configURL),
+              let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
+        else { return [] }
+
+        if let ids = extractEosIds(from: json) {
+            return ids
+        }
+
+        if let textConfig = json["text_config"] as? [String: Any],
+           let ids = extractEosIds(from: textConfig) {
+            return ids
+        }
+
+        return []
+    }
+
+    private func extractEosIds(from dict: [String: Any]) -> Set<Int>? {
+        if let id = dict["eos_token_id"] as? Int {
+            return [id]
+        }
+        if let ids = dict["eos_token_id"] as? [Int], !ids.isEmpty {
+            return Set(ids)
+        }
+        return nil
+    }
+
     private func buildToolSchema(from tool: ToolDefinition) -> ToolSpec {
         var properties: [String: [String: Any]] = [:]
         var required: [String] = []
@@ -125,6 +153,22 @@ class HybridLLM: HybridLLMSpec {
 
                 try Task.checkCancellation()
 
+                /*
+                 mlx-swift-lm only reads top-level eos_token_id from config.json.
+                 Models like Qwen3.5 nest it inside text_config, leaving the stop
+                 set empty. Parse it ourselves and patch the container.
+                */
+                let containerEos = await loadedContainer.configuration.eosTokenIds
+                if containerEos.isEmpty {
+                    let parsed = self.parseEosTokenIds(from: modelDir)
+                    if !parsed.isEmpty {
+                        log("Patching eosTokenIds from config: \(parsed)")
+                        await loadedContainer.update { ctx in
+                            ctx.configuration.eosTokenIds = parsed
+                        }
+                    }
+                }
+
                 let memoryAfterContainer = self.getMemoryUsage()
                 let gpuAfterContainer = self.getGPUMemoryUsage()
                 log("Model loaded - Host: \(memoryAfterContainer), GPU: \(gpuAfterContainer)")

package/lib/module/index.js

@@ -0,0 +1,9 @@
+"use strict";
+
+export { LLM } from "./llm.js";
+export { ModelManager } from "./modelManager.js";
+export { MLXModel, MLXModels, ModelFamily, ModelProvider } from "./models.js";
+export { createTool } from "./tool-utils.js";
+export { TTS } from "./tts.js";
+export { STT } from "./stt.js";
+//# sourceMappingURL=index.js.map

package/lib/module/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["LLM","ModelManager","MLXModel","MLXModels","ModelFamily","ModelProvider","createTool","TTS","STT"],"sourceRoot":"../../src","sources":["index.ts"],"mappings":";;AAAA,SACEA,GAAG,QAKE,UAAO;AACd,SAASC,YAAY,QAAQ,mBAAgB;AAC7C,SACEC,QAAQ,EACRC,SAAS,EACTC,WAAW,EAEXC,aAAa,QAGR,aAAU;AAqBjB,SAASC,UAAU,QAAqC,iBAAc;AACtE,SAASC,GAAG,QAAQ,UAAO;AAM3B,SAASC,GAAG,QAAQ,UAAO","ignoreList":[]}

package/lib/module/llm.js

@@ -0,0 +1,193 @@
+"use strict";
+
+import { NitroModules } from 'react-native-nitro-modules';
+let instance = null;
+function getInstance() {
+  if (!instance) {
+    instance = NitroModules.createHybridObject('LLM');
+  }
+  return instance;
+}
+
+/**
+ * LLM text generation using MLX on Apple Silicon.
+ *
+ * @example
+ * ```ts
+ * import { LLM } from 'react-native-nitro-mlx'
+ *
+ * // Load a model
+ * await LLM.load('mlx-community/Qwen3-0.6B-4bit', progress => {
+ *   console.log(`Loading: ${(progress * 100).toFixed(0)}%`)
+ * })
+ *
+ * // Stream a response
+ * await LLM.stream('Hello!', token => {
+ *   process.stdout.write(token)
+ * })
+ *
+ * // Get generation stats
+ * const stats = LLM.getLastGenerationStats()
+ * console.log(`${stats.tokensPerSecond} tokens/sec`)
+ * ```
+ */
+export const LLM = {
+  /**
+   * Load a model into memory. Downloads the model from HuggingFace if not already cached.
+   * @param modelId - HuggingFace model ID (e.g., 'mlx-community/Qwen3-0.6B-4bit')
+   * @param options - Callback invoked with loading progress (0-1)
+   */
+  load(modelId, options) {
+    return getInstance().load(modelId, options);
+  },
+  /**
+   * Generate a complete response for a prompt. Blocks until generation is complete.
+   * For streaming responses, use `stream()` instead.
+   * @param prompt - The input text to generate a response for
+   * @returns The complete generated text
+   */
+  generate(prompt) {
+    return getInstance().generate(prompt);
+  },
+  /**
+   * Stream a response token by token with optional tool calling support.
+   * Tools must be provided when loading the model via `load()` options.
+   * Tools are automatically executed when the model calls them.
+   * @param prompt - The input text to generate a response for
+   * @param onToken - Callback invoked for each generated token
+   * @param onToolCall - Optional callback invoked when a tool is called.
+   *   Receives the current tool call and an accumulated array of all tool calls so far.
+   * @returns The complete generated text
+   */
+  stream(prompt, onToken, onToolCall) {
+    const accumulatedToolCalls = [];
+    return getInstance().stream(prompt, onToken, (name, argsJson) => {
+      if (onToolCall) {
+        try {
+          const args = JSON.parse(argsJson);
+          const toolCall = {
+            name,
+            arguments: args
+          };
+          accumulatedToolCalls.push(toolCall);
+          onToolCall({
+            toolCall,
+            allToolCalls: [...accumulatedToolCalls]
+          });
+        } catch {
+          const toolCall = {
+            name,
+            arguments: {}
+          };
+          accumulatedToolCalls.push(toolCall);
+          onToolCall({
+            toolCall,
+            allToolCalls: [...accumulatedToolCalls]
+          });
+        }
+      }
+    });
+  },
+  /**
+   * Stream with typed events for thinking blocks and tool calls.
+   * Provides granular lifecycle events for UI updates.
+   *
+   * @param prompt - The input text
+   * @param onEvent - Callback receiving typed StreamEvent objects
+   * @returns Promise resolving to final content string (thinking content stripped)
+   *
+   * @example
+   * ```ts
+   * await LLM.streamWithEvents(prompt, (event) => {
+   *   switch (event.type) {
+   *     case 'token':
+   *       appendToContent(event.token)
+   *       break
+   *     case 'thinking_start':
+   *       showThinkingIndicator()
+   *       break
+   *     case 'thinking_chunk':
+   *       appendToThinking(event.chunk)
+   *       break
+   *     case 'tool_call_start':
+   *       showToolCallCard(event.name, event.arguments)
+   *       break
+   *   }
+   * })
+   * ```
+   */
+  streamWithEvents(prompt, onEvent) {
+    return getInstance().streamWithEvents(prompt, eventJson => {
+      try {
+        const event = JSON.parse(eventJson);
+        onEvent(event);
+      } catch {
+        // Silently ignore malformed events
+      }
+    });
+  },
+  /**
+   * Stop the current generation. Safe to call even if not generating.
+   */
+  stop() {
+    getInstance().stop();
+  },
+  /**
+   * Unload the current model and release memory.
+   * Call this when you're done with the model to free up memory.
+   */
+  unload() {
+    getInstance().unload();
+  },
+  /**
+   * Get statistics from the last generation.
+   * @returns Statistics including token count, tokens/sec (excluding tool execution), TTFT, total time, and tool execution time
+   */
+  getLastGenerationStats() {
+    return getInstance().getLastGenerationStats();
+  },
+  /**
+   * Get the message history if management is enabled.
+   * @returns Array of messages in the history
+   */
+  getHistory() {
+    return getInstance().getHistory();
+  },
+  /**
+   * Clear the message history.
+   */
+  clearHistory() {
+    getInstance().clearHistory();
+  },
+  /** Whether a model is currently loaded and ready for generation */
+  get isLoaded() {
+    return getInstance().isLoaded;
+  },
+  /** Whether text is currently being generated */
+  get isGenerating() {
+    return getInstance().isGenerating;
+  },
+  /** The ID of the currently loaded model, or empty string if none */
+  get modelId() {
+    return getInstance().modelId;
+  },
+  /** Enable debug logging to console */
+  get debug() {
+    return getInstance().debug;
+  },
+  set debug(value) {
+    getInstance().debug = value;
+  },
+  /**
+   * System prompt used when loading the model.
+   * Set this before calling `load()`. Changes require reloading the model.
+   * @default "You are a helpful assistant."
+   */
+  get systemPrompt() {
+    return getInstance().systemPrompt;
+  },
+  set systemPrompt(value) {
+    getInstance().systemPrompt = value;
+  }
+};
+//# sourceMappingURL=llm.js.map

package/lib/module/llm.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["NitroModules","instance","getInstance","createHybridObject","LLM","load","modelId","options","generate","prompt","stream","onToken","onToolCall","accumulatedToolCalls","name","argsJson","args","JSON","parse","toolCall","arguments","push","allToolCalls","streamWithEvents","onEvent","eventJson","event","stop","unload","getLastGenerationStats","getHistory","clearHistory","isLoaded","isGenerating","debug","value","systemPrompt"],"sourceRoot":"../../src","sources":["llm.ts"],"mappings":";;AAAA,SAASA,YAAY,QAAQ,4BAA4B;AAUzD,IAAIC,QAAwB,GAAG,IAAI;AAiBnC,SAASC,WAAWA,CAAA,EAAY;EAC9B,IAAI,CAACD,QAAQ,EAAE;IACbA,QAAQ,GAAGD,YAAY,CAACG,kBAAkB,CAAU,KAAK,CAAC;EAC5D;EACA,OAAOF,QAAQ;AACjB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMG,GAAG,GAAG;EACjB;AACF;AACA;AACA;AACA;EACEC,IAAIA,CAACC,OAAe,EAAEC,OAAuB,EAAiB;IAC5D,OAAOL,WAAW,CAAC,CAAC,CAACG,IAAI,CAACC,OAAO,EAAEC,OAAO,CAAC;EAC7C,CAAC;EAED;AACF;AACA;AACA;AACA;AACA;EACEC,QAAQA,CAACC,MAAc,EAAmB;IACxC,OAAOP,WAAW,CAAC,CAAC,CAACM,QAAQ,CAACC,MAAM,CAAC;EACvC,CAAC;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,MAAMA,CACJD,MAAc,EACdE,OAAgC,EAChCC,UAA6C,EAC5B;IACjB,MAAMC,oBAAoC,GAAG,EAAE;IAE/C,OAAOX,WAAW,CAAC,CAAC,CAACQ,MAAM,CAACD,MAAM,EAAEE,OAAO,EAAE,CAACG,IAAY,EAAEC,QAAgB,KAAK;MAC/E,IAAIH,UAAU,EAAE;QACd,IAAI;UACF,MAAMI,IAAI,GAAGC,IAAI,CAACC,KAAK,CAACH,QAAQ,CAA4B;UAC5D,MAAMI,QAAQ,GAAG;YAAEL,IAAI;YAAEM,SAAS,EAAEJ;UAAK,CAAC;UAC1CH,oBAAoB,CAACQ,IAAI,CAACF,QAAQ,CAAC;UACnCP,UAAU,CAAC;YACTO,QAAQ;YACRG,YAAY,EAAE,CAAC,GAAGT,oBAAoB;UACxC,CAAC,CAAC;QACJ,CAAC,CAAC,MAAM;UACN,MAAMM,QAAQ,GAAG;YAAEL,IAAI;YAAEM,SAAS,EAAE,CAAC;UAAE,CAAC;UACxCP,oBAAoB,CAACQ,IAAI,CAACF,QAAQ,CAAC;UACnCP,UAAU,CAAC;YACTO,QAAQ;YACRG,YAAY,EAAE,CAAC,GAAGT,oBAAoB;UACxC,CAAC,CAAC;QACJ;MACF;IACF,CAAC,CAAC;EACJ,CAAC;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEU,gBAAgBA,CAACd,MAAc,EAAEe,OAAsB,EAAmB;IACxE,OAAOtB,WAAW,CAAC,CAAC,CAACqB,gBAAgB,CAACd,MAAM,EAAGgB,SAAiB,IAAK;MACnE,IAAI;QACF,MAAMC,KAAK,GAAGT,IAAI,CAACC,KAAK,CAACO,SAAS,CAAgB;QAClDD,OAAO,CAACE,KAAK,CAAC;MAChB,CAAC,CAAC,MAAM;QACN;MAAA;IAEJ,CAAC,CAAC;EACJ,CAAC;EAED;AACF;AACA;EACEC,IAAIA,CAAA,EAAS;IACXzB,WAAW,CAAC,CAAC,CAACyB,IAAI,CAAC,CAAC;EACtB,CAAC;EAED;AACF;AACA;AACA;EACEC,MAAMA,CAAA,EAAS;IACb1B,WAAW,CAAC,CAAC,CAAC0B,MAAM,CAAC,CAAC;EACxB,CAAC;EAED;AACF;AACA;AACA;EACEC,sBAAsBA,CAAA,EAAoB;IACxC,OAAO3B,WAAW,CAAC,CAAC,CAAC2B,sBAAsB,CAAC,CAAC;EAC/C,CAAC;EAED;AACF;AACA;AACA;EACEC,UAAUA,CAAA,EAAc;IACtB,OAAO5B,WAAW,CAAC,CAAC,CAAC4B,UAAU,CAAC,CAAC;EACnC,CAAC;EAED;AACF;AACA;EACEC,YAAYA,CAAA,EAAS;IACnB7B,WAAW,CAAC,CAAC,CAAC6B,YAAY,CAAC,CAAC;EAC9B,CAAC;EAED;EACA,IAAIC,QAAQA,CAAA,EAAY;IACtB,OAAO9B,WAAW,CAAC,CAAC,CAAC8B,QAAQ;EAC/B,CAAC;EAED;EACA,IAAIC,YAAYA,CAAA,EAAY;IAC1B,OAAO/B,WAAW,CAAC,CAAC,CAAC+B,YAAY;EACnC,CAAC;EAED;EACA,IAAI3B,OAAOA,CAAA,EAAW;IACpB,OAAOJ,WAAW,CAAC,CAAC,CAACI,OAAO;EAC9B,CAAC;EAED;EACA,IAAI4B,KAAKA,CAAA,EAAY;IACnB,OAAOhC,WAAW,CAAC,CAAC,CAACgC,KAAK;EAC5B,CAAC;EAED,IAAIA,KAAKA,CAACC,KAAc,EAAE;IACxBjC,WAAW,CAAC,CAAC,CAACgC,KAAK,GAAGC,KAAK;EAC7B,CAAC;EAED;AACF;AACA;AACA;AACA;EACE,IAAIC,YAAYA,CAAA,EAAW;IACzB,OAAOlC,WAAW,CAAC,CAAC,CAACkC,YAAY;EACnC,CAAC;EAED,IAAIA,YAAYA,CAACD,KAAa,EAAE;IAC9BjC,WAAW,CAAC,CAAC,CAACkC,YAAY,GAAGD,KAAK;EACpC;AACF,CAAC","ignoreList":[]}

package/lib/module/modelManager.js

@@ -0,0 +1,79 @@
+"use strict";
+
+import { NitroModules } from 'react-native-nitro-modules';
+let instance = null;
+function getInstance() {
+  if (!instance) {
+    instance = NitroModules.createHybridObject('ModelManager');
+  }
+  return instance;
+}
+
+/**
+ * Manage MLX model downloads from HuggingFace.
+ *
+ * @example
+ * ```ts
+ * import { ModelManager } from 'react-native-nitro-mlx'
+ *
+ * // Download a model
+ * await ModelManager.download('mlx-community/Qwen3-0.6B-4bit', progress => {
+ *   console.log(`Downloading: ${(progress * 100).toFixed(0)}%`)
+ * })
+ *
+ * // Check if downloaded
+ * const isReady = await ModelManager.isDownloaded('mlx-community/Qwen3-0.6B-4bit')
+ *
+ * // List all downloaded models
+ * const models = await ModelManager.getDownloadedModels()
+ * ```
+ */
+export const ModelManager = {
+  /**
+   * Download a model from HuggingFace.
+   * @param modelId - HuggingFace model ID (e.g., 'mlx-community/Qwen3-0.6B-4bit')
+   * @param progressCallback - Callback invoked with download progress (0-1)
+   * @returns Path to the downloaded model directory
+   */
+  download(modelId, progressCallback) {
+    return getInstance().download(modelId, progressCallback);
+  },
+  /**
+   * Check if a model is already downloaded.
+   * @param modelId - HuggingFace model ID
+   * @returns True if the model is fully downloaded
+   */
+  isDownloaded(modelId) {
+    return getInstance().isDownloaded(modelId);
+  },
+  /**
+   * Get a list of all downloaded model IDs.
+   * @returns Array of model IDs that are available locally
+   */
+  getDownloadedModels() {
+    return getInstance().getDownloadedModels();
+  },
+  /**
+   * Delete a downloaded model to free up disk space.
+   * @param modelId - HuggingFace model ID
+   */
+  deleteModel(modelId) {
+    return getInstance().deleteModel(modelId);
+  },
+  /**
+   * Get the local filesystem path for a downloaded model.
+   * @param modelId - HuggingFace model ID
+   * @returns Absolute path to the model directory
+   */
+  getModelPath(modelId) {
+    return getInstance().getModelPath(modelId);
+  },
+  /** Enable debug logging to console */
+  get debug() {
+    return getInstance().debug;
+  },
+  set debug(value) {
+    getInstance().debug = value;
+  }
+};
+//# sourceMappingURL=modelManager.js.map

package/lib/module/modelManager.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["NitroModules","instance","getInstance","createHybridObject","ModelManager","download","modelId","progressCallback","isDownloaded","getDownloadedModels","deleteModel","getModelPath","debug","value"],"sourceRoot":"../../src","sources":["modelManager.ts"],"mappings":";;AAAA,SAASA,YAAY,QAAQ,4BAA4B;AAGzD,IAAIC,QAAiC,GAAG,IAAI;AAE5C,SAASC,WAAWA,CAAA,EAAqB;EACvC,IAAI,CAACD,QAAQ,EAAE;IACbA,QAAQ,GAAGD,YAAY,CAACG,kBAAkB,CAAmB,cAAc,CAAC;EAC9E;EACA,OAAOF,QAAQ;AACjB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMG,YAAY,GAAG;EAC1B;AACF;AACA;AACA;AACA;AACA;EACEC,QAAQA,CACNC,OAAe,EACfC,gBAA4C,EAC3B;IACjB,OAAOL,WAAW,CAAC,CAAC,CAACG,QAAQ,CAACC,OAAO,EAAEC,gBAAgB,CAAC;EAC1D,CAAC;EAED;AACF;AACA;AACA;AACA;EACEC,YAAYA,CAACF,OAAe,EAAoB;IAC9C,OAAOJ,WAAW,CAAC,CAAC,CAACM,YAAY,CAACF,OAAO,CAAC;EAC5C,CAAC;EAED;AACF;AACA;AACA;EACEG,mBAAmBA,CAAA,EAAsB;IACvC,OAAOP,WAAW,CAAC,CAAC,CAACO,mBAAmB,CAAC,CAAC;EAC5C,CAAC;EAED;AACF;AACA;AACA;EACEC,WAAWA,CAACJ,OAAe,EAAiB;IAC1C,OAAOJ,WAAW,CAAC,CAAC,CAACQ,WAAW,CAACJ,OAAO,CAAC;EAC3C,CAAC;EAED;AACF;AACA;AACA;AACA;EACEK,YAAYA,CAACL,OAAe,EAAmB;IAC7C,OAAOJ,WAAW,CAAC,CAAC,CAACS,YAAY,CAACL,OAAO,CAAC;EAC5C,CAAC;EAED;EACA,IAAIM,KAAKA,CAAA,EAAY;IACnB,OAAOV,WAAW,CAAC,CAAC,CAACU,KAAK;EAC5B,CAAC;EAED,IAAIA,KAAKA,CAACC,KAAc,EAAE;IACxBX,WAAW,CAAC,CAAC,CAACU,KAAK,GAAGC,KAAK;EAC7B;AACF,CAAC","ignoreList":[]}