tokenizers 0.4.4-arm64-darwin → 0.5.1-arm64-darwin

data/lib/tokenizers/decoders/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Decoders
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.15.2"
+    TOKENIZERS_VERSION = "0.20.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
@@ -67,7 +67,7 @@ module Tokenizers
         end
       end
 
-      options[:content_length_proc] = -> (_) { puts "Downloading..." }
+      options[:content_length_proc] = ->(_) { puts "Downloading..." }
 
       # string options are headers
       tempfile = URI.parse(url).open(headers.merge(options))

data/lib/tokenizers/pre_tokenizers/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module PreTokenizers
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.4.4"
+  VERSION = "0.5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.1
 platform: arm64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2024-08-13 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -23,7 +23,6 @@ files:
 - LICENSE.txt
 - README.md
 - lib/tokenizers.rb
-- lib/tokenizers/3.0/tokenizers.bundle
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/3.3/tokenizers.bundle
@@ -68,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
   - - "<"
     - !ruby/object:Gem::Version
       version: 3.4.dev