safetensors 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67fbb4b9469dd7bb195c8b89788868f051d80d2537a4d84dba20b91ddd23a79c
-  data.tar.gz: c9b784821f4821b8db148802e4e78f20b5c53732b8b6a12d089ec4c291cc1986
+  metadata.gz: 547c0781cd2c693db2a6605bca61004833aa668876f1512673ed7e5a44b6d30d
+  data.tar.gz: be088a6e2c725dfbc32a16c3864f8bc23ff4f58ee86fcf6c3388f64d98fb0223
 SHA512:
-  metadata.gz: b1eb7800dfed9d4589d40524825f98bb79d4360ef4a4f7cb69215d8e68b915b5f58ebc7f15b2f8b91e63153f7c1c59fcc0222117c8d04cd2716a9ea44495b54f
-  data.tar.gz: 0f2112f34fec093f483151fe6307b9983d942ea425b955bb9f8bc6adeef5febbb424f286f8385f3a51e21c0fdd0d1a2ddda30834069a5dd3f985a5b2eeeab8f6
+  metadata.gz: 270ea7030272173f04939528d34ae49d8d8a15f7ac51adb62fe6c93990e551257bce124c9bc983bbda70851322e66c4149f4483815f26b504a9528067b61dc1e
+  data.tar.gz: 7e4c3681d5443bb85d5226590d4dfe1b42a1c8ecf597ace1f136f89c6456070878cefc4f81ddf8247d8243f5c4fc20897af800c55e16aa4746d271d4cccdda53

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1 (2025-11-22)
+
+- Updated Safetensors to 0.7.0
+
 ## 0.2.0 (2025-06-23)
 
 - Updated Safetensors to 0.6.0

data/Cargo.lock

@@ -11,6 +11,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
 [[package]]
 name = "bindgen"
 version = "0.69.5"
@@ -69,12 +75,36 @@ version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
 
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
 [[package]]
 name = "glob"
 version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
+[[package]]
+name = "hashbrown"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+ "serde",
+]
+
 [[package]]
 name = "itertools"
 version = "0.12.1"
@@ -120,9 +150,9 @@ dependencies = [
 
 [[package]]
 name = "magnus"
-version = "0.7.1"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d87ae53030f3a22e83879e666cb94e58a7bdf31706878a0ba48752994146dab"
+checksum = "bd2ac6e71886be00ac34db92aa732c793c5107c95191805b9a1c7e70e6d342e0"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -132,9 +162,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
+checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -192,18 +222,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.116"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7059846f68396df83155779c75336ca24567741cb95256e6308c9fcc370e8dad"
+checksum = "f900d1ce4629a2ebffaf5de74bd8f9c1188d4c5ed406df02f97e22f77a006f44"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.116"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac217510df41b9ffc041573e68d7a02aaff770c49943c7494441c4b224b0ecd0"
+checksum = "ef1e9c857028f631056bcd6d88cec390c751e343ce2223ddb26d23eb4a151d59"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -216,9 +246,9 @@ dependencies = [
 
 [[package]]
 name = "rb-sys-env"
-version = "0.1.2"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
+checksum = "08f8d2924cf136a1315e2b4c7460a39f62ef11ee5d522df9b2750fab55b868b6"
 
 [[package]]
 name = "regex"
@@ -263,20 +293,21 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
 [[package]]
 name = "safetensors"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
  "magnus",
  "memmap2",
- "safetensors 0.6.0",
+ "safetensors 0.7.0",
  "serde_json",
 ]
 
 [[package]]
 name = "safetensors"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "617df84b8360fc910a687afe4be1c4660bf7c090d448d2fc27c781e852205f7c"
+checksum = "675656c1eabb620b921efea4f9199f97fc86e36dd6ffd1fbbe48d0f59a4987f5"
 dependencies = [
+ "hashbrown",
  "serde",
  "serde_json",
 ]

data/ext/safetensors/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "safetensors"
-version = "0.2.0"
+version = "0.2.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,7 +11,7 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = "0.7"
+magnus = "0.8"
 memmap2 = "0.5"
-safetensors = "=0.6.0"
+safetensors = "=0.7.0"
 serde_json = "1"

data/ext/safetensors/src/lib.rs

@@ -79,12 +79,16 @@ fn prepare(tensor_dict: &RHash) -> RbResult<HashMap<String, TensorView<'_>>> {
     Ok(tensors)
 }
 
-fn serialize(tensor_dict: RHash, metadata: Option<HashMap<String, String>>) -> RbResult<RString> {
+fn serialize(
+    ruby: &Ruby,
+    tensor_dict: RHash,
+    metadata: Option<HashMap<String, String>>,
+) -> RbResult<RString> {
     let tensors = prepare(&tensor_dict)?;
     let metadata_map = metadata.map(HashMap::from_iter);
     let out = safetensors::tensor::serialize(&tensors, metadata_map)
         .map_err(|e| SafetensorError::new_err(format!("Error while serializing: {e:?}")))?;
-    let rbbytes = RString::from_slice(&out);
+    let rbbytes = ruby.str_from_slice(&out);
     Ok(rbbytes)
 }
 
@@ -99,20 +103,20 @@ fn serialize_file(
     Ok(())
 }
 
-fn deserialize(bytes: RString) -> RbResult<RArray> {
+fn deserialize(ruby: &Ruby, bytes: RString) -> RbResult<RArray> {
     let safetensor = SafeTensors::deserialize(unsafe { bytes.as_slice() })
         .map_err(|e| SafetensorError::new_err(format!("Error while deserializing: {e:?}")))?;
 
     let tensors = safetensor.tensors();
-    let items = RArray::with_capacity(tensors.len());
+    let items = ruby.ary_new_capa(tensors.len());
 
     for (tensor_name, tensor) in tensors {
-        let rbshape = RArray::from_vec(tensor.shape().to_vec());
+        let rbshape = ruby.ary_from_vec(tensor.shape().to_vec());
         let rbdtype = format!("{:?}", tensor.dtype());
 
-        let rbdata = RString::from_slice(tensor.data());
+        let rbdata = ruby.str_from_slice(tensor.data());
 
-        let map = RHash::new();
+        let map = ruby.hash_new();
         map.aset("shape", rbshape)?;
         map.aset("dtype", rbdtype)?;
         map.aset("data", rbdata)?;
@@ -279,7 +283,7 @@ impl Open {
         Ok(keys)
     }
 
-    pub fn get_tensor(&self, name: &str) -> RbResult<Value> {
+    pub fn get_tensor(&self, ruby: &Ruby, name: &str) -> RbResult<Value> {
         let info = self.metadata.info(name).ok_or_else(|| {
             SafetensorError::new_err(format!("File does not contain tensor {name}",))
         })?;
@@ -289,9 +293,10 @@ impl Open {
                 let data =
                     &mmap[info.data_offsets.0 + self.offset..info.data_offsets.1 + self.offset];
 
-                let array: Value = RString::from_slice(data).into_value();
+                let array: Value = ruby.str_from_slice(data).as_value();
 
                 create_tensor(
+                    ruby,
                     &self.framework,
                     info.dtype,
                     &info.shape,
@@ -332,19 +337,19 @@ impl SafeOpen {
         self.inner()?.keys()
     }
 
-    pub fn get_tensor(&self, name: String) -> RbResult<Value> {
-        self.inner()?.get_tensor(&name)
+    pub fn get_tensor(ruby: &Ruby, rb_self: &Self, name: String) -> RbResult<Value> {
+        rb_self.inner()?.get_tensor(ruby, &name)
     }
 }
 
 fn create_tensor(
+    ruby: &Ruby,
     framework: &Framework,
     dtype: Dtype,
     shape: &[usize],
     array: Value,
     device: &Device,
 ) -> RbResult<Value> {
-    let ruby = Ruby::get().unwrap();
     let (module, is_numo): (RModule, bool) = match framework {
         Framework::Pytorch => (
             ruby.class_object()
@@ -360,7 +365,7 @@ fn create_tensor(
         ),
     };
 
-    let dtype = get_rbdtype(module, dtype, is_numo)?;
+    let dtype = get_rbdtype(ruby, module, dtype, is_numo)?;
     let shape = shape.to_vec();
     let tensor: Value = match framework {
         Framework::Pytorch => {
@@ -378,19 +383,19 @@ fn create_tensor(
     Ok(tensor)
 }
 
-fn get_rbdtype(_module: RModule, dtype: Dtype, is_numo: bool) -> RbResult<Value> {
-    let dtype: Value = if is_numo {
+fn get_rbdtype(ruby: &Ruby, _module: RModule, dtype: Dtype, is_numo: bool) -> RbResult<Symbol> {
+    let dtype: Symbol = if is_numo {
         match dtype {
-            Dtype::F64 => Symbol::new("DFloat").into_value(),
-            Dtype::F32 => Symbol::new("SFloat").into_value(),
-            Dtype::U64 => Symbol::new("UInt64").into_value(),
-            Dtype::I64 => Symbol::new("Int64").into_value(),
-            Dtype::U32 => Symbol::new("UInt32").into_value(),
-            Dtype::I32 => Symbol::new("Int32").into_value(),
-            Dtype::U16 => Symbol::new("UInt16").into_value(),
-            Dtype::I16 => Symbol::new("Int16").into_value(),
-            Dtype::U8 => Symbol::new("UInt8").into_value(),
-            Dtype::I8 => Symbol::new("Int8").into_value(),
+            Dtype::F64 => ruby.to_symbol("DFloat"),
+            Dtype::F32 => ruby.to_symbol("SFloat"),
+            Dtype::U64 => ruby.to_symbol("UInt64"),
+            Dtype::I64 => ruby.to_symbol("Int64"),
+            Dtype::U32 => ruby.to_symbol("UInt32"),
+            Dtype::I32 => ruby.to_symbol("Int32"),
+            Dtype::U16 => ruby.to_symbol("UInt16"),
+            Dtype::I16 => ruby.to_symbol("Int16"),
+            Dtype::U8 => ruby.to_symbol("UInt8"),
+            Dtype::I8 => ruby.to_symbol("Int8"),
             dtype => {
                 return Err(SafetensorError::new_err(format!(
                     "Dtype not understood: {dtype:?}"
@@ -399,21 +404,21 @@ fn get_rbdtype(_module: RModule, dtype: Dtype, is_numo: bool) -> RbResult<Value>
         }
     } else {
         match dtype {
-            Dtype::F64 => Symbol::new("float64").into_value(),
-            Dtype::F32 => Symbol::new("float32").into_value(),
-            Dtype::BF16 => Symbol::new("bfloat16").into_value(),
-            Dtype::F16 => Symbol::new("float16").into_value(),
-            Dtype::U64 => Symbol::new("uint64").into_value(),
-            Dtype::I64 => Symbol::new("int64").into_value(),
-            Dtype::U32 => Symbol::new("uint32").into_value(),
-            Dtype::I32 => Symbol::new("int32").into_value(),
-            Dtype::U16 => Symbol::new("uint16").into_value(),
-            Dtype::I16 => Symbol::new("int16").into_value(),
-            Dtype::U8 => Symbol::new("uint8").into_value(),
-            Dtype::I8 => Symbol::new("int8").into_value(),
-            Dtype::BOOL => Symbol::new("bool").into_value(),
-            Dtype::F8_E4M3 => Symbol::new("float8_e4m3fn").into_value(),
-            Dtype::F8_E5M2 => Symbol::new("float8_e5m2").into_value(),
+            Dtype::F64 => ruby.to_symbol("float64"),
+            Dtype::F32 => ruby.to_symbol("float32"),
+            Dtype::BF16 => ruby.to_symbol("bfloat16"),
+            Dtype::F16 => ruby.to_symbol("float16"),
+            Dtype::U64 => ruby.to_symbol("uint64"),
+            Dtype::I64 => ruby.to_symbol("int64"),
+            Dtype::U32 => ruby.to_symbol("uint32"),
+            Dtype::I32 => ruby.to_symbol("int32"),
+            Dtype::U16 => ruby.to_symbol("uint16"),
+            Dtype::I16 => ruby.to_symbol("int16"),
+            Dtype::U8 => ruby.to_symbol("uint8"),
+            Dtype::I8 => ruby.to_symbol("int8"),
+            Dtype::BOOL => ruby.to_symbol("bool"),
+            Dtype::F8_E4M3 => ruby.to_symbol("float8_e4m3fn"),
+            Dtype::F8_E5M2 => ruby.to_symbol("float8_e5m2"),
             dtype => {
                 return Err(SafetensorError::new_err(format!(
                     "Dtype not understood: {dtype:?}"

data/lib/safetensors/version.rb

@@ -1,3 +1,3 @@
 module Safetensors
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: safetensors
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -59,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Simple, safe way to store and distribute tensors
 test_files: []