tokenizers 0.3.2 → 0.4.0

data/lib/tokenizers/models/unigram.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Models
     class Unigram
-      def self.new(vocab: nil, unk_id: nil)
-        _new(vocab, unk_id)
+      def self.new(vocab: nil, unk_id: nil, byte_fallback: nil)
+        _new(vocab, unk_id, byte_fallback)
       end
     end
   end

data/lib/tokenizers/normalizers/prepend.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Prepend
+      def self.new(prepend: "▁")
+        _new(prepend)
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,15 @@
 # ext
 begin
-  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require_relative "tokenizers/tokenizers"
+  require "tokenizers/tokenizers"
 end
 
 # decoders
 require_relative "tokenizers/decoders/bpe_decoder"
 require_relative "tokenizers/decoders/ctc"
 require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/strip"
 require_relative "tokenizers/decoders/word_piece"
 
 # models
@@ -19,6 +20,7 @@ require_relative "tokenizers/models/unigram"
 
 # normalizers
 require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/prepend"
 require_relative "tokenizers/normalizers/strip"
 
 # pre-tokenizers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-07 00:00:00.000000000 Z
+date: 2023-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -56,6 +56,7 @@ files:
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
 - lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/strip.rb
 - lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
@@ -64,6 +65,7 @@ files:
 - lib/tokenizers/models/word_level.rb
 - lib/tokenizers/models/word_piece.rb
 - lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/prepend.rb
 - lib/tokenizers/normalizers/strip.rb
 - lib/tokenizers/pre_tokenizers/byte_level.rb
 - lib/tokenizers/pre_tokenizers/digits.rb
@@ -91,14 +93,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby