tokenizers 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ff4d1ad7b56010f603ead7a4794c003c5294e50f1b33de62c8089ddf150d5ad
-  data.tar.gz: 295aaabb720971f2ddcc832ab0d5deedf1e0ed8dab03aca96ac1d396b5723de7
+  metadata.gz: 11fdc53989bc2285096bc5eb4a971426e153375ada43bc8c51d2a191b42fe02d
+  data.tar.gz: 92d4d8bef2c4013d5cf0d55bff30bc070ca6e59430f87b96f9147075216a1c1d
 SHA512:
-  metadata.gz: e14207004cddeef40590229ea2c8a9bf54e5c5b75cdbcdd32cd6f23c24feb8544fcabe86fa9bced32cb41f2581ee0df4d36ed2b6a58ef2fc668aa33c270659df
-  data.tar.gz: b2bb202c8c37bdd0d14ca64be147e99b224128c1461f56761f1d58d9326b40768e7b903bbbb4c2a0363bd4b1c9ef5a66be53210ad801d5e45c7d86dd0945bd82
+  metadata.gz: 7c91c33078b6c5b23a6080908fa184e2222922f35bd6e1c439329525f4554f939cf98ca4dcd3017d858cdb6ec2a96f6671ce5dc425ca643d482b49b04af00f4c
+  data.tar.gz: 74aad6c458792570ace93107aacccb8a9df7b61228dd062bd7bcaf015a53d4f6b4b1fcace09c69bc959bd9b04598d315344d4edc7e6abbe1603f3b92d29ab711

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-## 0.3.0 (2022-02-07)
+## 0.3.2 (2023-03-06)
+
+- Added precompiled gem for Linux x86-64 MUSL
+
+## 0.3.1 (2023-02-08)
+
+- Fixed error with Ruby 2.7
+
+## 0.3.0 (2023-02-07)
 
 - Added support for training tokenizers
 - Added more methods to `Tokenizer`
@@ -7,20 +15,20 @@
 - Changed `encode` method to include special tokens by default
 - Changed how offsets are calculated for strings with multibyte characters
 
-## 0.2.3 (2022-01-22)
+## 0.2.3 (2023-01-22)
 
 - Added `add_special_tokens` option to `encode` method
 - Added warning about `encode` method including special tokens by default in 0.3.0
 - Added more methods to `Encoding`
 - Fixed error with precompiled gem on Mac ARM
 
-## 0.2.2 (2022-01-15)
+## 0.2.2 (2023-01-15)
 
 - Added precompiled gem for Linux ARM
 - Added `from_file` method
 - Fixed error with precompiled gem on Linux x86-64
 
-## 0.2.1 (2022-01-12)
+## 0.2.1 (2023-01-12)
 
 - Added support for Ruby 3.2
 

data/Cargo.lock

@@ -353,7 +353,8 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
 [[package]]
 name = "magnus"
 version = "0.5.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af37419a942477f606d227d0e6e92f3b68458bfc68fec3bc2629df6a2c1ccdf9"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -362,8 +363,9 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.3.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85aa71c9891b2732ff1157e1860a1ee578459fd25811fd3d72cc6e32b3fbdfea"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -552,22 +554,25 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc8945662df8083245deda89e236647173cc7ad750f481ddcd7bbfd3afe3fa5e"
+checksum = "e8fe617bad8e88fd7e5d6f432e35f09e5f94144dfb8e8ee4adde82fb920dc59b"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.64"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8c3cdf9edc3908ee1555b7a1bca58ee1b499439b32cd1c1ec3e66736a8df48"
+checksum = "007e63597f91c711cbb299e60fecbdb6f5ad4a066d6a20c81943893f1584c895"
 dependencies = [
  "bindgen",
+ "lazy_static",
+ "quote",
  "regex",
  "shell-words",
+ "syn",
 ]
 
 [[package]]
@@ -711,7 +716,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "magnus",
  "onig",

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -10,7 +10,7 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { git = "https://github.com/matsadler/magnus" }
+magnus = "0.5"
 onig = { version = "6.0", default-features = false }
 serde = { version = "1.0", features = ["rc", "derive"] }
 

data/lib/tokenizers/from_pretrained.rb

@@ -44,7 +44,7 @@ module Tokenizers
     def cached_path(cache_dir, url, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
-      meta = meta_paths.map { |f| JSON.load_file(f) }.max_by { |m| m["creation_time"] }
+      meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
       etag = meta["etag"] if meta
 
       if etag

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-08 00:00:00.000000000 Z
+date: 2023-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys