tokenizers 0.2.3-arm64-darwin → 0.3.1-arm64-darwin

data/README.md

@@ -40,6 +40,10 @@ Load a tokenizer from files
 tokenizer = Tokenizers::CharBPETokenizer.new("vocab.json", "merges.txt")
 ```
 
+## Training
+
+Check out the [Quicktour](https://huggingface.co/docs/tokenizers/quicktour) and equivalent [Ruby code](https://github.com/ankane/tokenizers-ruby/blob/master/test/quicktour_test.rb#L8)
+
 ## History
 
 View the [changelog](https://github.com/ankane/tokenizers-ruby/blob/master/CHANGELOG.md)

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -1,11 +1,14 @@
 module Tokenizers
   class CharBPETokenizer
-    def initialize(vocab, merges)
-      @tokenizer = Tokenizer.new(BPE.new(vocab, merges))
-      @tokenizer.add_special_tokens(["<unk>"])
-      @tokenizer.normalizer = BertNormalizer.new
-      @tokenizer.pre_tokenizer = BertPreTokenizer.new
-      @tokenizer.decoder = BPEDecoder.new
+    def initialize(vocab, merges, unk_token: "<unk>", suffix: "</w>")
+      @tokenizer =
+        Tokenizer.new(
+          Models::BPE._from_file(vocab, merges, {unk_token: unk_token, end_of_word_suffix: suffix})
+        )
+      @tokenizer.add_special_tokens([unk_token])
+      @tokenizer.normalizer = Normalizers::BertNormalizer.new
+      @tokenizer.pre_tokenizer = PreTokenizers::BertPreTokenizer.new
+      @tokenizer.decoder = Decoders::BPEDecoder.new
     end
 
     def encode(text, **options)

data/lib/tokenizers/decoders/bpe_decoder.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class BPEDecoder
+      def self.new(suffix: "</w>")
+        _new(suffix)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/ctc.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class CTC
+      def self.new(pad_token: "<pad>", word_delimiter_token: "|", cleanup: true)
+        _new(pad_token, word_delimiter_token, cleanup)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/metaspace.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Metaspace
+      def self.new(replacement: "\u2581", add_prefix_space: true)
+        _new(replacement, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/word_piece.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class WordPiece
+      def self.new(prefix: '##', cleanup: true)
+        _new(prefix, cleanup)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -44,7 +44,7 @@ module Tokenizers
     def cached_path(cache_dir, url, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
-      meta = meta_paths.map { |f| JSON.load_file(f) }.max_by { |m| m["creation_time"] }
+      meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
       etag = meta["etag"] if meta
 
       if etag
@@ -57,7 +57,7 @@ module Tokenizers
 
       tempfile =
         begin
-          URI.open(url, options)
+          URI.parse(url).open(options)
         rescue OpenURI::HTTPError => e
           if e.message == "304 Not Modified"
             return resource_path

data/lib/tokenizers/models/bpe.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class BPE
+      def self.new(vocab: nil, merges: nil, **kwargs)
+        _new(vocab, merges, kwargs)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/unigram.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class Unigram
+      def self.new(vocab: nil, unk_id: nil)
+        _new(vocab, unk_id)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/word_level.rb

@@ -0,0 +1,13 @@
+module Tokenizers
+  module Models
+    class WordLevel
+      def self.new(vocab: nil, unk_token: nil)
+        _new(vocab, unk_token)
+      end
+
+      def self.from_file(vocab, unk_token: nil)
+        _from_file(vocab, unk_token)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/word_piece.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class WordPiece
+      def self.new(vocab: nil, **kwargs)
+        _new(vocab, kwargs)
+      end
+    end
+  end
+end

data/lib/tokenizers/normalizers/bert_normalizer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class BertNormalizer
+      def self.new(clean_text: true, handle_chinese_chars: true, strip_accents: nil, lowercase: true)
+        _new(clean_text, handle_chinese_chars, strip_accents, lowercase)
+      end
+    end
+  end
+end

data/lib/tokenizers/normalizers/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Strip
+      def self.new(left: true, right: true)
+        _new(left, right)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/byte_level.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class ByteLevel
+      def self.new(add_prefix_space: true, use_regex: true)
+        _new(add_prefix_space, use_regex)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/digits.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Digits
+      def self.new(individual_digits: false)
+        _new(individual_digits)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/metaspace.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Metaspace
+      def self.new(replacement: "\u2581", add_prefix_space: true)
+        _new(replacement, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/punctuation.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Punctuation
+      def self.new(behavior: "isolated")
+        _new(behavior)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/split.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Split
+      def self.new(pattern, behavior, invert: false)
+        _new(pattern, behavior, invert)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/byte_level.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class ByteLevel
+      def self.new(trim_offsets: true)
+        _new(trim_offsets)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/roberta_processing.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class RobertaProcessing
+      def self.new(sep, cls, trim_offsets: true, add_prefix_space: true)
+        _new(sep, cls, trim_offsets, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/template_processing.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class TemplateProcessing
+      def self.new(single: nil, pair: nil, special_tokens: nil)
+        _new(single, pair, special_tokens)
+      end
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -1,12 +1,45 @@
 module Tokenizers
   class Tokenizer
-    # TODO change add_special_tokens default to true in 0.3.0
-    def encode(sequence, add_special_tokens: nil)
-      if add_special_tokens.nil?
-        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
-        add_special_tokens = false
-      end
-      _encode(sequence, add_special_tokens)
+    extend FromPretrained
+
+    def to_s(pretty: false)
+      _to_s(pretty)
+    end
+
+    def save(path, pretty: false)
+      _save(path, pretty)
+    end
+
+    def encode(sequence, pair = nil, is_pretokenized: false, add_special_tokens: true)
+      _encode(sequence, pair, is_pretokenized, add_special_tokens)
+    end
+
+    def encode_batch(input, is_pretokenized: false, add_special_tokens: true)
+      _encode_batch(input, is_pretokenized, add_special_tokens)
+    end
+
+    def decode(ids, skip_special_tokens: true)
+      _decode(ids, skip_special_tokens)
+    end
+
+    def decode_batch(sequences, skip_special_tokens: true)
+      _decode_batch(sequences, skip_special_tokens)
+    end
+
+    def enable_padding(**options)
+      _enable_padding(options)
+    end
+
+    def enable_truncation(max_length, **options)
+      _enable_truncation(max_length, options)
+    end
+
+    def vocab(with_added_tokens: true)
+      _vocab(with_added_tokens)
+    end
+
+    def vocab_size(with_added_tokens: true)
+      _vocab_size(with_added_tokens)
     end
   end
 end

data/lib/tokenizers/trainers/bpe_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class BpeTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/unigram_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class UnigramTrainer
+      def self.new(vocab_size: 8000,
+                   show_progress: true,
+                   special_tokens: [],
+                   initial_alphabet: [],
+                   shrinking_factor: 0.75,
+                   unk_token: nil,
+                   max_piece_length: 16,
+                   n_sub_iterations: 2)
+
+        _new({
+          vocab_size: vocab_size,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          initial_alphabet: initial_alphabet,
+          shrinking_factor: shrinking_factor,
+          unk_token: unk_token,
+          max_piece_length: max_piece_length,
+          n_sub_iterations: n_sub_iterations
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_level_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class WordLevelTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_piece_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class WordPieceTrainer
+      def self.new(vocab_size: 30000,
+                   min_frequency: 0,
+                   show_progress: true,
+                   special_tokens: [],
+                   limit_alphabet: nil,
+                   initial_alphabet: [],
+                   continuing_subword_prefix: "##",
+                   end_of_word_suffix: nil)
+
+        _new({
+          vocab_size: vocab_size,
+          min_frequency: min_frequency,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          limit_alphabet: limit_alphabet,
+          initial_alphabet: initial_alphabet,
+          continuing_subword_prefix: continuing_subword_prefix,
+          end_of_word_suffix: end_of_word_suffix
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.3"
+  VERSION = "0.3.1"
 end

data/lib/tokenizers.rb

@@ -5,7 +5,41 @@ rescue LoadError
   require_relative "tokenizers/tokenizers"
 end
 
-# modules
+# decoders
+require_relative "tokenizers/decoders/bpe_decoder"
+require_relative "tokenizers/decoders/ctc"
+require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/word_piece"
+
+# models
+require_relative "tokenizers/models/bpe"
+require_relative "tokenizers/models/word_level"
+require_relative "tokenizers/models/word_piece"
+require_relative "tokenizers/models/unigram"
+
+# normalizers
+require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/strip"
+
+# pre-tokenizers
+require_relative "tokenizers/pre_tokenizers/byte_level"
+require_relative "tokenizers/pre_tokenizers/digits"
+require_relative "tokenizers/pre_tokenizers/metaspace"
+require_relative "tokenizers/pre_tokenizers/punctuation"
+require_relative "tokenizers/pre_tokenizers/split"
+
+# processors
+require_relative "tokenizers/processors/byte_level"
+require_relative "tokenizers/processors/roberta_processing"
+require_relative "tokenizers/processors/template_processing"
+
+# trainers
+require_relative "tokenizers/trainers/bpe_trainer"
+require_relative "tokenizers/trainers/unigram_trainer"
+require_relative "tokenizers/trainers/word_level_trainer"
+require_relative "tokenizers/trainers/word_piece_trainer"
+
+# other
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"
@@ -15,5 +49,11 @@ require_relative "tokenizers/version"
 module Tokenizers
   class Error < StandardError; end
 
-  extend FromPretrained
+  def self.from_pretrained(...)
+    Tokenizer.from_pretrained(...)
+  end
+
+  def self.from_file(...)
+    Tokenizer.from_file(...)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.1
 platform: arm64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-22 00:00:00.000000000 Z
+date: 2023-02-09 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -28,9 +28,31 @@ files:
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/decoders/bpe_decoder.rb
+- lib/tokenizers/decoders/ctc.rb
+- lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/models/bpe.rb
+- lib/tokenizers/models/unigram.rb
+- lib/tokenizers/models/word_level.rb
+- lib/tokenizers/models/word_piece.rb
+- lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/strip.rb
+- lib/tokenizers/pre_tokenizers/byte_level.rb
+- lib/tokenizers/pre_tokenizers/digits.rb
+- lib/tokenizers/pre_tokenizers/metaspace.rb
+- lib/tokenizers/pre_tokenizers/punctuation.rb
+- lib/tokenizers/pre_tokenizers/split.rb
+- lib/tokenizers/processors/byte_level.rb
+- lib/tokenizers/processors/roberta_processing.rb
+- lib/tokenizers/processors/template_processing.rb
 - lib/tokenizers/tokenizer.rb
+- lib/tokenizers/trainers/bpe_trainer.rb
+- lib/tokenizers/trainers/unigram_trainer.rb
+- lib/tokenizers/trainers/word_level_trainer.rb
+- lib/tokenizers/trainers/word_piece_trainer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses: