tokenizers 0.3.0-x86_64-linux → 0.3.2-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dd7a6e80756ca265d9152f6fa371e3debcc65ad511a9049d496bfe9265dd2f7
-  data.tar.gz: 36bcd374aa6d797fdb44014cc6c6c080042dff9a90c1b28ccebed0fb8d2d68d1
+  metadata.gz: 5c255ccee12a9f089e4c55c80c8874d2eb236bd8e2d2bc5b07a56b69ff553415
+  data.tar.gz: fe58ce9b34f3220aec3ae2f8090caf0cc519b3dacc187328a7b364b38f3d430f
 SHA512:
-  metadata.gz: 54b887046decc5dbd22549309b9087f1c79751269202ac1aa6ea04616388dcbbada02b2b926aa3649b9208747811250aad0f56d5cbd751966697b3e4f9c7447c
-  data.tar.gz: 8d7259138060315e6dd159a8b9b5622e20a366d72e9728e5f0c8a3a986a39b98f388dc779075ef873b18e5648139a1cbbb17e105172fc760967dbe62442ebc10
+  metadata.gz: 6745dcdda9eb9aecbf7fc31755107de76110ab6ed9d1ffcedbc63e6e034c7d4276ba6de959763ba4463dcbb2ec8cd1783680f558a733977c6799cd427064b6af
+  data.tar.gz: 959f49d956aafee7556a38b724b812b7c49ff2e22f55b0770ff069bafd7dcf63419d6dc33b65cef71da5b45b020534d6905d6770e95ce6142b9f57e59c3582bc

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-## 0.3.0 (2022-02-07)
+## 0.3.2 (2023-03-06)
+
+- Added precompiled gem for Linux x86-64 MUSL
+
+## 0.3.1 (2023-02-08)
+
+- Fixed error with Ruby 2.7
+
+## 0.3.0 (2023-02-07)
 
 - Added support for training tokenizers
 - Added more methods to `Tokenizer`
@@ -7,20 +15,20 @@
 - Changed `encode` method to include special tokens by default
 - Changed how offsets are calculated for strings with multibyte characters
 
-## 0.2.3 (2022-01-22)
+## 0.2.3 (2023-01-22)
 
 - Added `add_special_tokens` option to `encode` method
 - Added warning about `encode` method including special tokens by default in 0.3.0
 - Added more methods to `Encoding`
 - Fixed error with precompiled gem on Mac ARM
 
-## 0.2.2 (2022-01-15)
+## 0.2.2 (2023-01-15)
 
 - Added precompiled gem for Linux ARM
 - Added `from_file` method
 - Fixed error with precompiled gem on Linux x86-64
 
-## 0.2.1 (2022-01-12)
+## 0.2.1 (2023-01-12)
 
 - Added support for Ruby 3.2
 

data/Cargo.lock

@@ -353,7 +353,8 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
 [[package]]
 name = "magnus"
 version = "0.5.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -362,8 +363,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.3.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -552,22 +554,25 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc8945662df8083245deda89e236647173cc7ad750f481ddcd7bbfd3afe3fa5e"
+checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8c3cdf9edc3908ee1555b7a1bca58ee1b499439b32cd1c1ec3e66736a8df48"
+checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
 dependencies = [
  "bindgen",
+ "lazy_static",
+ "quote",
  "regex",
  "shell-words",
+ "syn",
 ]
 
 [[package]]
@@ -711,7 +716,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -154,7 +154,7 @@ magnus v0.5.0
 https://github.com/matsadler/magnus
 MIT
 
-magnus-macros v0.3.0
+magnus-macros v0.4.0
 https://github.com/matsadler/magnus
 MIT
 
@@ -242,11 +242,11 @@ rayon-core v1.10.2
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.64
+rb-sys v0.9.65
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.64
+rb-sys-build v0.9.65
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -7795,32 +7795,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
-================================================================================
-magnus magnus-macros/LICENSE
-================================================================================
-
-MIT License
-
-Copyright (c) 2022, 2021 Matthew Sadler
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 magnus-macros LICENSE
 ================================================================================

data/lib/tokenizers/from_pretrained.rb

@@ -44,7 +44,7 @@ module Tokenizers
     def cached_path(cache_dir, url, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
-      meta = meta_paths.map { |f| JSON.load_file(f) }.max_by { |m| m["creation_time"] }
+      meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
       etag = meta["etag"] if meta
 
       if etag

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: x86_64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-08 00:00:00.000000000 Z
+date: 2023-03-07 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org