tokenizers 0.3.0-x86_64-darwin → 0.3.2-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7fa86d3c3577bafde32ef67fd5a3888c81e010b4a7ea9658b0d46176a58dad7
-  data.tar.gz: 5b8e89123770e84310d76f3ce6fc2eb0b4ca24c676e2fd61262ba5c78d8b3fe9
+  metadata.gz: a79dd7bc52f269fdb6dd01849a2e0bbdf916d92e0bcfb39a8f8a04d70d0e8c76
+  data.tar.gz: 1455ff72f0bbdfbe48bad1287086ad6fc7c3b7663a74ff320896762a3b13e05f
 SHA512:
-  metadata.gz: 97525233b05ed90f540953ad3a4dc1d74aad3263a7cf913976644c7ab803e48e44fbcb37ee3f1c6734b2530ce1a5b734dcd7eb5d95a9ae51c430729d42d755d9
-  data.tar.gz: 171b94a79c7c76ac9c2e701c3ece322a3dfb220c653736a3e9bfa8c1302b5bdf0478a8c444bd50541baf16f8267879fbc0e284290028656a068844aae2b48336
+  metadata.gz: 4439fa4958c377d9b936bfca4af93de721a8b3b7ff6994b612b023b5a95629b3467bafd43f30d93e179c798bf1c6fcf31bcb99b6bde72cfee556fb9f4e74e68c
+  data.tar.gz: af98eb6353bdcc291a0fd2d666525f1640c797b54a59010ecff7a1a7ae9356643980be20b6f6dd61e4ed0589c6e221cdfd12982cf2db9d0a0565ae00a25cbdf1

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-## 0.3.0 (2022-02-07)
+## 0.3.2 (2023-03-06)
+
+- Added precompiled gem for Linux x86-64 MUSL
+
+## 0.3.1 (2023-02-08)
+
+- Fixed error with Ruby 2.7
+
+## 0.3.0 (2023-02-07)
 
 - Added support for training tokenizers
 - Added more methods to `Tokenizer`
@@ -7,20 +15,20 @@
 - Changed `encode` method to include special tokens by default
 - Changed how offsets are calculated for strings with multibyte characters
 
-## 0.2.3 (2022-01-22)
+## 0.2.3 (2023-01-22)
 
 - Added `add_special_tokens` option to `encode` method
 - Added warning about `encode` method including special tokens by default in 0.3.0
 - Added more methods to `Encoding`
 - Fixed error with precompiled gem on Mac ARM
 
-## 0.2.2 (2022-01-15)
+## 0.2.2 (2023-01-15)
 
 - Added precompiled gem for Linux ARM
 - Added `from_file` method
 - Fixed error with precompiled gem on Linux x86-64
 
-## 0.2.1 (2022-01-12)
+## 0.2.1 (2023-01-12)
 
 - Added support for Ruby 3.2
 

data/Cargo.lock

@@ -353,7 +353,8 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
 [[package]]
 name = "magnus"
 version = "0.5.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -362,8 +363,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.3.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -552,22 +554,25 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc8945662df8083245deda89e236647173cc7ad750f481ddcd7bbfd3afe3fa5e"
+checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8c3cdf9edc3908ee1555b7a1bca58ee1b499439b32cd1c1ec3e66736a8df48"
+checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
 dependencies = [
  "bindgen",
+ "lazy_static",
+ "quote",
  "regex",
  "shell-words",
+ "syn",
 ]
 
 [[package]]
@@ -711,7 +716,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "magnus",
  "onig",

data/LICENSE-THIRD-PARTY.txt

@@ -154,7 +154,7 @@ magnus v0.5.0
 https://github.com/matsadler/magnus
 MIT
 
-magnus-macros v0.3.0
+magnus-macros v0.4.0
 https://github.com/matsadler/magnus
 MIT
 
@@ -242,11 +242,11 @@ rayon-core v1.10.2
 https://github.com/rayon-rs/rayon
 MIT OR Apache-2.0
 
-rb-sys v0.9.64
+rb-sys v0.9.65
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
-rb-sys-build v0.9.64
+rb-sys-build v0.9.65
 https://github.com/oxidize-rb/rb-sys
 MIT OR Apache-2.0
 
@@ -7795,32 +7795,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
-================================================================================
-magnus magnus-macros/LICENSE
-================================================================================
-
-MIT License
-
-Copyright (c) 2022, 2021 Matthew Sadler
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 magnus-macros LICENSE
 ================================================================================

data/lib/tokenizers/from_pretrained.rb

@@ -44,7 +44,7 @@ module Tokenizers
     def cached_path(cache_dir, url, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
-      meta = meta_paths.map { |f| JSON.load_file(f) }.max_by { |m| m["creation_time"] }
+      meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
       etag = meta["etag"] if meta
 
       if etag

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: x86_64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-08 00:00:00.000000000 Z
+date: 2023-03-07 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org