tokenizers 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ff4d1ad7b56010f603ead7a4794c003c5294e50f1b33de62c8089ddf150d5ad
-  data.tar.gz: 295aaabb720971f2ddcc832ab0d5deedf1e0ed8dab03aca96ac1d396b5723de7
+  metadata.gz: b298d2a912a3d7fe32b9dc60036822b39ff463ad9739fec9ca00809922c6f45c
+  data.tar.gz: 8491e0c8532d9eefe5ae1c0940f57ae3986480fb9460897d7612e8937d97e384
 SHA512:
-  metadata.gz: e14207004cddeef40590229ea2c8a9bf54e5c5b75cdbcdd32cd6f23c24feb8544fcabe86fa9bced32cb41f2581ee0df4d36ed2b6a58ef2fc668aa33c270659df
-  data.tar.gz: b2bb202c8c37bdd0d14ca64be147e99b224128c1461f56761f1d58d9326b40768e7b903bbbb4c2a0363bd4b1c9ef5a66be53210ad801d5e45c7d86dd0945bd82
+  metadata.gz: 3616d92731fe9c7e4214166392b05734b10dec0e004106c01c756cab544491523b596b5d0a44ef8a47708719fa32839b86a1038a31104fae6d3de3a2e17ccca9
+  data.tar.gz: 88a375508473b97ae0b30dfb12a2c930a20477b003e5f2977a670904dab077ab4e2771b98dd3a446fc9b9c417e886f3b7355bd7eec527eb898251c9c60a0337c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.1 (2022-02-08)
+
+- Fixed error with Ruby 2.7
+
 ## 0.3.0 (2022-02-07)
 
 - Added support for training tokenizers

data/Cargo.lock

@@ -711,7 +711,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 dependencies = [
  "magnus",
  "onig",

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.2.3"
+version = "0.3.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"

data/lib/tokenizers/from_pretrained.rb

@@ -44,7 +44,7 @@ module Tokenizers
     def cached_path(cache_dir, url, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
-      meta = meta_paths.map { |f| JSON.load_file(f) }.max_by { |m| m["creation_time"] }
+      meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
       etag = meta["etag"] if meta
 
       if etag

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-08 00:00:00.000000000 Z
+date: 2023-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys