tokenizers 0.3.2-x86_64-linux-musl → 0.4.0-x86_64-linux-musl

data/lib/tokenizers/decoders/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Strip
+      def self.new(content: " ", start: 0, stop: 0)
+        _new(content, start, stop)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.13.2"
+    TOKENIZERS_VERSION = "0.14.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/models/unigram.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Models
     class Unigram
-      def self.new(vocab: nil, unk_id: nil)
-        _new(vocab, unk_id)
+      def self.new(vocab: nil, unk_id: nil, byte_fallback: nil)
+        _new(vocab, unk_id, byte_fallback)
       end
     end
   end

data/lib/tokenizers/normalizers/prepend.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Prepend
+      def self.new(prepend: "▁")
+        _new(prepend)
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,15 @@
 # ext
 begin
-  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require_relative "tokenizers/tokenizers"
+  require "tokenizers/tokenizers"
 end
 
 # decoders
 require_relative "tokenizers/decoders/bpe_decoder"
 require_relative "tokenizers/decoders/ctc"
 require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/strip"
 require_relative "tokenizers/decoders/word_piece"
 
 # models
@@ -19,6 +20,7 @@ require_relative "tokenizers/models/unigram"
 
 # normalizers
 require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/prepend"
 require_relative "tokenizers/normalizers/strip"
 
 # pre-tokenizers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: x86_64-linux-musl
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-07 00:00:00.000000000 Z
+date: 2023-09-21 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -23,7 +23,6 @@ files:
 - LICENSE.txt
 - README.md
 - lib/tokenizers.rb
-- lib/tokenizers/2.7/tokenizers.so
 - lib/tokenizers/3.0/tokenizers.so
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
@@ -31,6 +30,7 @@ files:
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
 - lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/strip.rb
 - lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
@@ -39,6 +39,7 @@ files:
 - lib/tokenizers/models/word_level.rb
 - lib/tokenizers/models/word_piece.rb
 - lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/prepend.rb
 - lib/tokenizers/normalizers/strip.rb
 - lib/tokenizers/pre_tokenizers/byte_level.rb
 - lib/tokenizers/pre_tokenizers/digits.rb
@@ -66,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
   - - "<"
     - !ruby/object:Gem::Version
       version: 3.3.dev