mlx-ruby-lm 0.30.7.1

data/lib/mlx_lm/weight_utils.rb

@@ -0,0 +1,170 @@
+require "safetensors"
+
+module MlxLm
+  module WeightUtils
+    module_function
+
+    DTYPE_UNPACK = {
+      "F32" => ["e*", :float32],
+      "F16" => ["e*", :float16],  # handled specially
+      "BF16" => ["e*", :bfloat16], # handled specially
+      "F64" => ["E*", :float32],
+      "I8" => ["c*", :int8],
+      "I16" => ["s<*", :int16],
+      "I32" => ["l<*", :int32],
+      "I64" => ["q<*", :int64],
+      "U8" => ["C*", :uint8],
+      "U16" => ["S<*", :uint16],
+      "U32" => ["L<*", :uint32],
+    }.freeze
+
+    # Load a single safetensors file, returning { name => MLX::Core::Array }
+    def load_safetensors(path)
+      mx = MLX::Core
+      raw = File.binread(path)
+      tensors = Safetensors.deserialize(raw)
+      result = {}
+      tensors.each do |name, info|
+        result[name] = _tensor_to_mlx(info, mx)
+      end
+      result
+    end
+
+    # Load all model*.safetensors shards from a directory
+    def load_sharded_safetensors(directory)
+      pattern = File.join(directory, "model*.safetensors")
+      files = Dir.glob(pattern).sort
+
+      if files.empty?
+        raise "No safetensors found in #{directory}"
+      end
+
+      weights = {}
+      files.each do |f|
+        weights.merge!(load_safetensors(f))
+      end
+      weights
+    end
+
+    def _tensor_to_mlx(info, mx)
+      shape = info["shape"]
+      dtype_str = info["dtype"]
+      data = info["data"]
+
+      # For F32/float32, unpack as little-endian floats
+      if dtype_str == "F32" || dtype_str == "float32"
+        values = data.unpack("e*")
+        mx.array(values).reshape(shape)
+      elsif dtype_str == "F16"
+        # 16-bit float: unpack as uint16, create array as float32, then view as float16
+        values = data.unpack("S<*")
+        mx.array(values, dtype: mx.uint16).view(mx.float16).reshape(shape)
+      elsif dtype_str == "BF16"
+        values = data.unpack("S<*")
+        mx.array(values, dtype: mx.uint16).view(mx.bfloat16).reshape(shape)
+      elsif dtype_str == "I32" || dtype_str == "int32"
+        values = data.unpack("l<*")
+        mx.array(values, dtype: mx.int32).reshape(shape)
+      elsif dtype_str == "I64"
+        values = data.unpack("q<*")
+        mx.array(values, dtype: mx.int64).reshape(shape)
+      elsif dtype_str == "U8"
+        values = data.unpack("C*")
+        mx.array(values, dtype: mx.uint8).reshape(shape)
+      else
+        # Fallback: try F32
+        values = data.unpack("e*")
+        mx.array(values).reshape(shape)
+      end
+    end
+
+    # Convert a flat weight dict like:
+    #   { "model.layers.0.weight" => tensor, ... }
+    # into a nested hash/array structure like:
+    #   { "model" => { "layers" => [{ "weight" => tensor }] } }
+    #
+    # Numeric path segments become array indices.
+    def tree_unflatten(flat)
+      root = {}
+
+      flat.each do |dotted_key, value|
+        parts = dotted_key.split(".")
+        node = root
+
+        parts.each_with_index do |part, i|
+          is_last = (i == parts.length - 1)
+          next_part = parts[i + 1] unless is_last
+
+          if is_last
+            _set_in_node(node, part, value)
+          else
+            node = _ensure_child(node, part, next_part)
+          end
+        end
+      end
+
+      _finalize_arrays(root)
+    end
+
+    # Set a value in the current node (hash or array)
+    def _set_in_node(node, key, value)
+      if node.is_a?(Hash)
+        if key.match?(/\A\d+\z/)
+          idx = key.to_i
+          node[idx] = value
+        else
+          node[key] = value
+        end
+      end
+    end
+
+    # Ensure a child container exists for the given key
+    def _ensure_child(node, key, next_key)
+      numeric_key = key.match?(/\A\d+\z/)
+      next_is_numeric = next_key && next_key.match?(/\A\d+\z/)
+
+      if numeric_key
+        idx = key.to_i
+        existing = node[idx]
+        if existing.nil?
+          child = next_is_numeric ? {} : {}
+          node[idx] = child
+          child
+        else
+          existing
+        end
+      else
+        existing = node[key]
+        if existing.nil?
+          child = {}
+          node[key] = child
+          child
+        else
+          existing
+        end
+      end
+    end
+
+    # Recursively convert hashes with all-integer keys into arrays
+    def _finalize_arrays(node)
+      return node unless node.is_a?(Hash)
+
+      # First, recurse into children
+      node.each do |k, v|
+        node[k] = _finalize_arrays(v)
+      end
+
+      # Check if all keys are integers (indicating this should be an array)
+      if node.keys.all? { |k| k.is_a?(Integer) }
+        max_idx = node.keys.max
+        arr = Array.new(max_idx + 1)
+        node.each { |k, v| arr[k] = v }
+        arr
+      else
+        node
+      end
+    end
+
+    private_class_method :_set_in_node, :_ensure_child, :_finalize_arrays, :_tensor_to_mlx
+  end
+end

data/lib/mlx_lm.rb

@@ -0,0 +1,135 @@
+require_relative "mlx_lm/version"
+require_relative "mlx_lm/model_args"
+require_relative "mlx_lm/weight_utils"
+require_relative "mlx_lm/config"
+require_relative "mlx_lm/tokenizer_utils"
+require_relative "mlx_lm/sample_utils"
+require_relative "mlx_lm/models"
+require_relative "mlx_lm/models/cache"
+require_relative "mlx_lm/models/activations"
+require_relative "mlx_lm/models/bitlinear_layers"
+require_relative "mlx_lm/models/bitnet"
+require_relative "mlx_lm/models/gated_delta"
+require_relative "mlx_lm/models/pipeline"
+require_relative "mlx_lm/models/rope_utils"
+require_relative "mlx_lm/models/ssm"
+require_relative "mlx_lm/models/mla"
+require_relative "mlx_lm/models/llama"
+require_relative "mlx_lm/models/gemma"
+require_relative "mlx_lm/models/qwen2"
+require_relative "mlx_lm/models/qwen2_vl"
+require_relative "mlx_lm/models/qwen"
+require_relative "mlx_lm/models/qwen3"
+require_relative "mlx_lm/models/qwen3_vl"
+require_relative "mlx_lm/models/qwen3_5"
+require_relative "mlx_lm/models/qwen3_5_moe"
+require_relative "mlx_lm/models/phi"
+require_relative "mlx_lm/models/phi3"
+require_relative "mlx_lm/models/exaone"
+require_relative "mlx_lm/models/exaone4"
+require_relative "mlx_lm/models/glm"
+require_relative "mlx_lm/models/glm4"
+require_relative "mlx_lm/models/helium"
+require_relative "mlx_lm/models/olmo"
+require_relative "mlx_lm/models/seed_oss"
+require_relative "mlx_lm/models/starcoder2"
+require_relative "mlx_lm/models/stablelm"
+require_relative "mlx_lm/models/cohere"
+require_relative "mlx_lm/models/cohere2"
+require_relative "mlx_lm/models/pixtral"
+require_relative "mlx_lm/models/gemma2"
+require_relative "mlx_lm/models/gemma3_text"
+require_relative "mlx_lm/models/gemma3"
+require_relative "mlx_lm/models/gemma3n"
+require_relative "mlx_lm/models/granite"
+require_relative "mlx_lm/models/granitemoe"
+require_relative "mlx_lm/models/olmo2"
+require_relative "mlx_lm/models/olmoe"
+require_relative "mlx_lm/models/openelm"
+require_relative "mlx_lm/models/gpt_neox"
+require_relative "mlx_lm/models/switch_layers"
+require_relative "mlx_lm/models/qwen3_moe"
+require_relative "mlx_lm/models/qwen3_vl_moe"
+require_relative "mlx_lm/models/mixtral"
+require_relative "mlx_lm/models/phixtral"
+require_relative "mlx_lm/models/mistral3"
+require_relative "mlx_lm/models/minicpm"
+require_relative "mlx_lm/models/minicpm3"
+require_relative "mlx_lm/models/nanochat"
+require_relative "mlx_lm/models/smollm3"
+require_relative "mlx_lm/models/lfm2"
+require_relative "mlx_lm/models/lfm2_vl"
+require_relative "mlx_lm/models/deepseek"
+require_relative "mlx_lm/models/deepseek_v2"
+require_relative "mlx_lm/models/deepseek_v3"
+require_relative "mlx_lm/models/deepseek_v32"
+require_relative "mlx_lm/models/glm_moe_dsa"
+require_relative "mlx_lm/models/kimi_k25"
+require_relative "mlx_lm/models/kimi_vl"
+require_relative "mlx_lm/models/internlm2"
+require_relative "mlx_lm/models/internlm3"
+require_relative "mlx_lm/models/telechat3"
+require_relative "mlx_lm/models/olmo3"
+require_relative "mlx_lm/models/gpt2"
+require_relative "mlx_lm/models/gpt_bigcode"
+require_relative "mlx_lm/models/nemotron"
+require_relative "mlx_lm/models/apertus"
+require_relative "mlx_lm/models/youtu_llm"
+require_relative "mlx_lm/models/ernie4_5"
+require_relative "mlx_lm/models/ernie4_5_moe"
+require_relative "mlx_lm/models/baichuan_m1"
+require_relative "mlx_lm/models/solar_open"
+require_relative "mlx_lm/models/lille_130m"
+require_relative "mlx_lm/models/mimo"
+require_relative "mlx_lm/models/qwen2_moe"
+require_relative "mlx_lm/models/phimoe"
+require_relative "mlx_lm/models/llama4_text"
+require_relative "mlx_lm/models/plamo"
+require_relative "mlx_lm/models/mamba"
+require_relative "mlx_lm/models/mamba2"
+require_relative "mlx_lm/models/hunyuan_v1_dense"
+require_relative "mlx_lm/models/dbrx"
+require_relative "mlx_lm/models/klear"
+require_relative "mlx_lm/models/iquestloopcoder"
+require_relative "mlx_lm/models/phi3small"
+require_relative "mlx_lm/models/dots1"
+require_relative "mlx_lm/models/llama4"
+require_relative "mlx_lm/models/ministral3"
+require_relative "mlx_lm/models/hunyuan"
+require_relative "mlx_lm/models/gpt_oss"
+require_relative "mlx_lm/models/mimo_v2_flash"
+require_relative "mlx_lm/models/lfm2_moe"
+require_relative "mlx_lm/models/afmoe"
+require_relative "mlx_lm/models/bailing_moe"
+require_relative "mlx_lm/models/exaone_moe"
+require_relative "mlx_lm/models/glm4_moe"
+require_relative "mlx_lm/models/minimax"
+require_relative "mlx_lm/models/nemotron_nas"
+require_relative "mlx_lm/models/recurrent_gemma"
+require_relative "mlx_lm/models/step3p5"
+require_relative "mlx_lm/models/afm7"
+require_relative "mlx_lm/models/bailing_moe_linear"
+require_relative "mlx_lm/models/falcon_h1"
+require_relative "mlx_lm/models/glm4_moe_lite"
+require_relative "mlx_lm/models/granitemoehybrid"
+require_relative "mlx_lm/models/jamba"
+require_relative "mlx_lm/models/kimi_linear"
+require_relative "mlx_lm/models/longcat_flash"
+require_relative "mlx_lm/models/longcat_flash_ngram"
+require_relative "mlx_lm/models/nemotron_h"
+require_relative "mlx_lm/models/plamo2"
+require_relative "mlx_lm/models/qwen3_next"
+require_relative "mlx_lm/models/rwkv7"
+require_relative "mlx_lm/generate"
+require_relative "mlx_lm/quantize"
+require_relative "mlx_lm/load_utils"
+require_relative "mlx_lm/tuner/lora"
+require_relative "mlx_lm/chat_template"
+require_relative "mlx_lm/cli"
+require_relative "mlx_lm/server"
+require_relative "mlx_lm/perplexity"
+require_relative "mlx_lm/benchmark"
+require_relative "mlx_lm/convert_utils"
+
+module MlxLm
+end

metadata

@@ -0,0 +1,272 @@
+--- !ruby/object:Gem::Specification
+name: mlx-ruby-lm
+version: !ruby/object:Gem::Version
+  version: 0.30.7.1
+platform: ruby
+authors:
+- Alex Skryl
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mlx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.30.7.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.30.7.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: safetensors
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: tokenizers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.20'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: A Ruby port of mlx-lm providing large language model inference, quantization,
+  LoRA fine-tuning, and an OpenAI-compatible server built on the mlx gem. Supports
+  Llama, Gemma, Qwen2, Phi3, Mixtral, DeepSeek, and many more architectures.
+email:
+- rut216@gmail.com
+executables:
+- mlx_lm
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- exe/mlx_lm
+- lib/mlx_lm.rb
+- lib/mlx_lm/benchmark.rb
+- lib/mlx_lm/chat_template.rb
+- lib/mlx_lm/cli.rb
+- lib/mlx_lm/config.rb
+- lib/mlx_lm/convert_utils.rb
+- lib/mlx_lm/generate.rb
+- lib/mlx_lm/load_utils.rb
+- lib/mlx_lm/model_args.rb
+- lib/mlx_lm/models.rb
+- lib/mlx_lm/models/activations.rb
+- lib/mlx_lm/models/afm7.rb
+- lib/mlx_lm/models/afmoe.rb
+- lib/mlx_lm/models/apertus.rb
+- lib/mlx_lm/models/baichuan_m1.rb
+- lib/mlx_lm/models/bailing_moe.rb
+- lib/mlx_lm/models/bailing_moe_linear.rb
+- lib/mlx_lm/models/bitlinear_layers.rb
+- lib/mlx_lm/models/bitnet.rb
+- lib/mlx_lm/models/cache.rb
+- lib/mlx_lm/models/cohere.rb
+- lib/mlx_lm/models/cohere2.rb
+- lib/mlx_lm/models/dbrx.rb
+- lib/mlx_lm/models/deepseek.rb
+- lib/mlx_lm/models/deepseek_v2.rb
+- lib/mlx_lm/models/deepseek_v3.rb
+- lib/mlx_lm/models/deepseek_v32.rb
+- lib/mlx_lm/models/dots1.rb
+- lib/mlx_lm/models/ernie4_5.rb
+- lib/mlx_lm/models/ernie4_5_moe.rb
+- lib/mlx_lm/models/exaone.rb
+- lib/mlx_lm/models/exaone4.rb
+- lib/mlx_lm/models/exaone_moe.rb
+- lib/mlx_lm/models/falcon_h1.rb
+- lib/mlx_lm/models/gated_delta.rb
+- lib/mlx_lm/models/gemma.rb
+- lib/mlx_lm/models/gemma2.rb
+- lib/mlx_lm/models/gemma3.rb
+- lib/mlx_lm/models/gemma3_text.rb
+- lib/mlx_lm/models/gemma3n.rb
+- lib/mlx_lm/models/glm.rb
+- lib/mlx_lm/models/glm4.rb
+- lib/mlx_lm/models/glm4_moe.rb
+- lib/mlx_lm/models/glm4_moe_lite.rb
+- lib/mlx_lm/models/glm_moe_dsa.rb
+- lib/mlx_lm/models/gpt2.rb
+- lib/mlx_lm/models/gpt_bigcode.rb
+- lib/mlx_lm/models/gpt_neox.rb
+- lib/mlx_lm/models/gpt_oss.rb
+- lib/mlx_lm/models/granite.rb
+- lib/mlx_lm/models/granitemoe.rb
+- lib/mlx_lm/models/granitemoehybrid.rb
+- lib/mlx_lm/models/helium.rb
+- lib/mlx_lm/models/hunyuan.rb
+- lib/mlx_lm/models/hunyuan_v1_dense.rb
+- lib/mlx_lm/models/internlm2.rb
+- lib/mlx_lm/models/internlm3.rb
+- lib/mlx_lm/models/iquestloopcoder.rb
+- lib/mlx_lm/models/jamba.rb
+- lib/mlx_lm/models/kimi_k25.rb
+- lib/mlx_lm/models/kimi_linear.rb
+- lib/mlx_lm/models/kimi_vl.rb
+- lib/mlx_lm/models/klear.rb
+- lib/mlx_lm/models/lfm2.rb
+- lib/mlx_lm/models/lfm2_moe.rb
+- lib/mlx_lm/models/lfm2_vl.rb
+- lib/mlx_lm/models/lille_130m.rb
+- lib/mlx_lm/models/llama.rb
+- lib/mlx_lm/models/llama4.rb
+- lib/mlx_lm/models/llama4_text.rb
+- lib/mlx_lm/models/longcat_flash.rb
+- lib/mlx_lm/models/longcat_flash_ngram.rb
+- lib/mlx_lm/models/mamba.rb
+- lib/mlx_lm/models/mamba2.rb
+- lib/mlx_lm/models/mimo.rb
+- lib/mlx_lm/models/mimo_v2_flash.rb
+- lib/mlx_lm/models/minicpm.rb
+- lib/mlx_lm/models/minicpm3.rb
+- lib/mlx_lm/models/minimax.rb
+- lib/mlx_lm/models/ministral3.rb
+- lib/mlx_lm/models/mistral3.rb
+- lib/mlx_lm/models/mixtral.rb
+- lib/mlx_lm/models/mla.rb
+- lib/mlx_lm/models/nanochat.rb
+- lib/mlx_lm/models/nemotron.rb
+- lib/mlx_lm/models/nemotron_h.rb
+- lib/mlx_lm/models/nemotron_nas.rb
+- lib/mlx_lm/models/olmo.rb
+- lib/mlx_lm/models/olmo2.rb
+- lib/mlx_lm/models/olmo3.rb
+- lib/mlx_lm/models/olmoe.rb
+- lib/mlx_lm/models/openelm.rb
+- lib/mlx_lm/models/phi.rb
+- lib/mlx_lm/models/phi3.rb
+- lib/mlx_lm/models/phi3small.rb
+- lib/mlx_lm/models/phimoe.rb
+- lib/mlx_lm/models/phixtral.rb
+- lib/mlx_lm/models/pipeline.rb
+- lib/mlx_lm/models/pixtral.rb
+- lib/mlx_lm/models/plamo.rb
+- lib/mlx_lm/models/plamo2.rb
+- lib/mlx_lm/models/qwen.rb
+- lib/mlx_lm/models/qwen2.rb
+- lib/mlx_lm/models/qwen2_moe.rb
+- lib/mlx_lm/models/qwen2_vl.rb
+- lib/mlx_lm/models/qwen3.rb
+- lib/mlx_lm/models/qwen3_5.rb
+- lib/mlx_lm/models/qwen3_5_moe.rb
+- lib/mlx_lm/models/qwen3_moe.rb
+- lib/mlx_lm/models/qwen3_next.rb
+- lib/mlx_lm/models/qwen3_vl.rb
+- lib/mlx_lm/models/qwen3_vl_moe.rb
+- lib/mlx_lm/models/recurrent_gemma.rb
+- lib/mlx_lm/models/rope_utils.rb
+- lib/mlx_lm/models/rwkv7.rb
+- lib/mlx_lm/models/seed_oss.rb
+- lib/mlx_lm/models/smollm3.rb
+- lib/mlx_lm/models/solar_open.rb
+- lib/mlx_lm/models/ssm.rb
+- lib/mlx_lm/models/stablelm.rb
+- lib/mlx_lm/models/starcoder2.rb
+- lib/mlx_lm/models/step3p5.rb
+- lib/mlx_lm/models/switch_layers.rb
+- lib/mlx_lm/models/telechat3.rb
+- lib/mlx_lm/models/youtu_llm.rb
+- lib/mlx_lm/perplexity.rb
+- lib/mlx_lm/quantize.rb
+- lib/mlx_lm/sample_utils.rb
+- lib/mlx_lm/server.rb
+- lib/mlx_lm/tokenizer_utils.rb
+- lib/mlx_lm/tuner/lora.rb
+- lib/mlx_lm/version.rb
+- lib/mlx_lm/weight_utils.rb
+homepage: https://github.com/skryl/mlx-ruby-lm
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/skryl/mlx-ruby-lm
+  source_code_uri: https://github.com/skryl/mlx-ruby-lm
+  changelog_uri: https://github.com/skryl/mlx-ruby-lm/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: LLM inference and fine-tuning on MLX for Ruby
+test_files: []