tokenizers 0.3.2-arm64-darwin → 0.3.3-arm64-darwin

data/lib/tokenizers/decoders/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Strip
+      def self.new(content: " ", start: 0, stop: 0)
+        _new(content, start, stop)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.13.2"
+    TOKENIZERS_VERSION = "0.13.3"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/normalizers/prepend.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Prepend
+      def self.new(prepend: "▁")
+        _new(prepend)
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.2"
+  VERSION = "0.3.3"
 end

data/lib/tokenizers.rb

@@ -9,6 +9,7 @@ end
 require_relative "tokenizers/decoders/bpe_decoder"
 require_relative "tokenizers/decoders/ctc"
 require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/strip"
 require_relative "tokenizers/decoders/word_piece"
 
 # models
@@ -19,6 +20,7 @@ require_relative "tokenizers/models/unigram"
 
 # normalizers
 require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/prepend"
 require_relative "tokenizers/normalizers/strip"
 
 # pre-tokenizers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: arm64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-07 00:00:00.000000000 Z
+date: 2023-04-09 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -31,6 +31,7 @@ files:
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
 - lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/strip.rb
 - lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
@@ -39,6 +40,7 @@ files:
 - lib/tokenizers/models/word_level.rb
 - lib/tokenizers/models/word_piece.rb
 - lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/prepend.rb
 - lib/tokenizers/normalizers/strip.rb
 - lib/tokenizers/pre_tokenizers/byte_level.rb
 - lib/tokenizers/pre_tokenizers/digits.rb