tokenizers 0.2.3 → 0.3.0

data/ext/tokenizers/src/utils/mod.rs

@@ -0,0 +1,5 @@
+mod normalization;
+mod regex;
+
+pub use normalization::*;
+pub use regex::*;

data/ext/tokenizers/src/utils/normalization.rs

@@ -0,0 +1,85 @@
+use super::regex::{regex, RbRegex};
+use crate::RbResult;
+use magnus::{exception, Error, TryConvert, Value};
+use tk::normalizer::SplitDelimiterBehavior;
+use tk::pattern::Pattern;
+
+#[derive(Clone)]
+pub enum RbPattern<'p> {
+    Str(String),
+    Regex(&'p RbRegex),
+}
+
+impl TryConvert for RbPattern<'_> {
+    fn try_convert(obj: Value) -> RbResult<Self> {
+        if obj.is_kind_of(regex()) {
+            Ok(RbPattern::Regex(obj.try_convert()?))
+        } else {
+            Ok(RbPattern::Str(obj.try_convert()?))
+        }
+    }
+}
+
+impl Pattern for RbPattern<'_> {
+    fn find_matches(&self, inside: &str) -> tk::Result<Vec<(tk::Offsets, bool)>> {
+        match self {
+            RbPattern::Str(s) => {
+                let mut chars = s.chars();
+                if let (Some(c), None) = (chars.next(), chars.next()) {
+                    c.find_matches(inside)
+                } else {
+                    s.find_matches(inside)
+                }
+            }
+            RbPattern::Regex(_r) => {
+                todo!()
+            }
+        }
+    }
+}
+
+impl From<RbPattern<'_>> for tk::normalizers::replace::ReplacePattern {
+    fn from(pattern: RbPattern<'_>) -> Self {
+        match pattern {
+            RbPattern::Str(s) => Self::String(s),
+            RbPattern::Regex(_r) => todo!(),
+        }
+    }
+}
+
+impl From<RbPattern<'_>> for tk::pre_tokenizers::split::SplitPattern {
+    fn from(pattern: RbPattern<'_>) -> Self {
+        match pattern {
+            RbPattern::Str(s) => Self::String(s),
+            RbPattern::Regex(_r) => todo!(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RbSplitDelimiterBehavior(pub SplitDelimiterBehavior);
+
+impl TryConvert for RbSplitDelimiterBehavior {
+    fn try_convert(obj: Value) -> RbResult<Self> {
+        let s = obj.try_convert::<String>()?;
+
+        Ok(Self(match s.as_str() {
+            "removed" => Ok(SplitDelimiterBehavior::Removed),
+            "isolated" => Ok(SplitDelimiterBehavior::Isolated),
+            "merged_with_previous" => Ok(SplitDelimiterBehavior::MergedWithPrevious),
+            "merged_with_next" => Ok(SplitDelimiterBehavior::MergedWithNext),
+            "contiguous" => Ok(SplitDelimiterBehavior::Contiguous),
+            _ => Err(Error::new(
+                exception::arg_error(),
+                "Wrong value for SplitDelimiterBehavior, expected one of: \
+                `removed, isolated, merged_with_previous, merged_with_next, contiguous`",
+            )),
+        }?))
+    }
+}
+
+impl From<RbSplitDelimiterBehavior> for SplitDelimiterBehavior {
+    fn from(v: RbSplitDelimiterBehavior) -> Self {
+        v.0
+    }
+}

data/ext/tokenizers/src/utils/regex.rs

@@ -0,0 +1,22 @@
+use onig::Regex;
+use magnus::{exception, memoize, Error, Module, RClass};
+use crate::{module, RbResult};
+
+#[magnus::wrap(class = "Tokenizers::Regex")]
+pub struct RbRegex {
+    pub inner: Regex,
+    pub pattern: String,
+}
+
+impl RbRegex {
+    pub fn new(s: String) -> RbResult<Self> {
+        Ok(Self {
+            inner: Regex::new(&s).map_err(|e| Error::new(exception::runtime_error(), e.description().to_owned()))?,
+            pattern: s,
+        })
+    }
+}
+
+pub fn regex() -> RClass {
+    *memoize!(RClass: module().const_get("Regex").unwrap())
+}

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -1,11 +1,14 @@
 module Tokenizers
   class CharBPETokenizer
-    def initialize(vocab, merges)
-      @tokenizer = Tokenizer.new(BPE.new(vocab, merges))
-      @tokenizer.add_special_tokens(["<unk>"])
-      @tokenizer.normalizer = BertNormalizer.new
-      @tokenizer.pre_tokenizer = BertPreTokenizer.new
-      @tokenizer.decoder = BPEDecoder.new
+    def initialize(vocab, merges, unk_token: "<unk>", suffix: "</w>")
+      @tokenizer =
+        Tokenizer.new(
+          Models::BPE._from_file(vocab, merges, {unk_token: unk_token, end_of_word_suffix: suffix})
+        )
+      @tokenizer.add_special_tokens([unk_token])
+      @tokenizer.normalizer = Normalizers::BertNormalizer.new
+      @tokenizer.pre_tokenizer = PreTokenizers::BertPreTokenizer.new
+      @tokenizer.decoder = Decoders::BPEDecoder.new
     end
 
     def encode(text, **options)

data/lib/tokenizers/decoders/bpe_decoder.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class BPEDecoder
+      def self.new(suffix: "</w>")
+        _new(suffix)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/ctc.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class CTC
+      def self.new(pad_token: "<pad>", word_delimiter_token: "|", cleanup: true)
+        _new(pad_token, word_delimiter_token, cleanup)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/metaspace.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Metaspace
+      def self.new(replacement: "\u2581", add_prefix_space: true)
+        _new(replacement, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/decoders/word_piece.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class WordPiece
+      def self.new(prefix: '##', cleanup: true)
+        _new(prefix, cleanup)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -57,7 +57,7 @@ module Tokenizers
 
       tempfile =
         begin
-          URI.open(url, options)
+          URI.parse(url).open(options)
         rescue OpenURI::HTTPError => e
           if e.message == "304 Not Modified"
             return resource_path

data/lib/tokenizers/models/bpe.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class BPE
+      def self.new(vocab: nil, merges: nil, **kwargs)
+        _new(vocab, merges, kwargs)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/unigram.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class Unigram
+      def self.new(vocab: nil, unk_id: nil)
+        _new(vocab, unk_id)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/word_level.rb

@@ -0,0 +1,13 @@
+module Tokenizers
+  module Models
+    class WordLevel
+      def self.new(vocab: nil, unk_token: nil)
+        _new(vocab, unk_token)
+      end
+
+      def self.from_file(vocab, unk_token: nil)
+        _from_file(vocab, unk_token)
+      end
+    end
+  end
+end

data/lib/tokenizers/models/word_piece.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Models
+    class WordPiece
+      def self.new(vocab: nil, **kwargs)
+        _new(vocab, kwargs)
+      end
+    end
+  end
+end

data/lib/tokenizers/normalizers/bert_normalizer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class BertNormalizer
+      def self.new(clean_text: true, handle_chinese_chars: true, strip_accents: nil, lowercase: true)
+        _new(clean_text, handle_chinese_chars, strip_accents, lowercase)
+      end
+    end
+  end
+end

data/lib/tokenizers/normalizers/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Strip
+      def self.new(left: true, right: true)
+        _new(left, right)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/byte_level.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class ByteLevel
+      def self.new(add_prefix_space: true, use_regex: true)
+        _new(add_prefix_space, use_regex)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/digits.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Digits
+      def self.new(individual_digits: false)
+        _new(individual_digits)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/metaspace.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Metaspace
+      def self.new(replacement: "\u2581", add_prefix_space: true)
+        _new(replacement, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/punctuation.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Punctuation
+      def self.new(behavior: "isolated")
+        _new(behavior)
+      end
+    end
+  end
+end

data/lib/tokenizers/pre_tokenizers/split.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module PreTokenizers
+    class Split
+      def self.new(pattern, behavior, invert: false)
+        _new(pattern, behavior, invert)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/byte_level.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class ByteLevel
+      def self.new(trim_offsets: true)
+        _new(trim_offsets)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/roberta_processing.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class RobertaProcessing
+      def self.new(sep, cls, trim_offsets: true, add_prefix_space: true)
+        _new(sep, cls, trim_offsets, add_prefix_space)
+      end
+    end
+  end
+end

data/lib/tokenizers/processors/template_processing.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Processors
+    class TemplateProcessing
+      def self.new(single: nil, pair: nil, special_tokens: nil)
+        _new(single, pair, special_tokens)
+      end
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -1,12 +1,45 @@
 module Tokenizers
   class Tokenizer
-    # TODO change add_special_tokens default to true in 0.3.0
-    def encode(sequence, add_special_tokens: nil)
-      if add_special_tokens.nil?
-        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
-        add_special_tokens = false
-      end
-      _encode(sequence, add_special_tokens)
+    extend FromPretrained
+
+    def to_s(pretty: false)
+      _to_s(pretty)
+    end
+
+    def save(path, pretty: false)
+      _save(path, pretty)
+    end
+
+    def encode(sequence, pair = nil, is_pretokenized: false, add_special_tokens: true)
+      _encode(sequence, pair, is_pretokenized, add_special_tokens)
+    end
+
+    def encode_batch(input, is_pretokenized: false, add_special_tokens: true)
+      _encode_batch(input, is_pretokenized, add_special_tokens)
+    end
+
+    def decode(ids, skip_special_tokens: true)
+      _decode(ids, skip_special_tokens)
+    end
+
+    def decode_batch(sequences, skip_special_tokens: true)
+      _decode_batch(sequences, skip_special_tokens)
+    end
+
+    def enable_padding(**options)
+      _enable_padding(options)
+    end
+
+    def enable_truncation(max_length, **options)
+      _enable_truncation(max_length, options)
+    end
+
+    def vocab(with_added_tokens: true)
+      _vocab(with_added_tokens)
+    end
+
+    def vocab_size(with_added_tokens: true)
+      _vocab_size(with_added_tokens)
     end
   end
 end

data/lib/tokenizers/trainers/bpe_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class BpeTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/unigram_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class UnigramTrainer
+      def self.new(vocab_size: 8000,
+                   show_progress: true,
+                   special_tokens: [],
+                   initial_alphabet: [],
+                   shrinking_factor: 0.75,
+                   unk_token: nil,
+                   max_piece_length: 16,
+                   n_sub_iterations: 2)
+
+        _new({
+          vocab_size: vocab_size,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          initial_alphabet: initial_alphabet,
+          shrinking_factor: shrinking_factor,
+          unk_token: unk_token,
+          max_piece_length: max_piece_length,
+          n_sub_iterations: n_sub_iterations
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_level_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class WordLevelTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_piece_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class WordPieceTrainer
+      def self.new(vocab_size: 30000,
+                   min_frequency: 0,
+                   show_progress: true,
+                   special_tokens: [],
+                   limit_alphabet: nil,
+                   initial_alphabet: [],
+                   continuing_subword_prefix: "##",
+                   end_of_word_suffix: nil)
+
+        _new({
+          vocab_size: vocab_size,
+          min_frequency: min_frequency,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          limit_alphabet: limit_alphabet,
+          initial_alphabet: initial_alphabet,
+          continuing_subword_prefix: continuing_subword_prefix,
+          end_of_word_suffix: end_of_word_suffix
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

data/lib/tokenizers.rb

@@ -5,7 +5,41 @@ rescue LoadError
   require_relative "tokenizers/tokenizers"
 end
 
-# modules
+# decoders
+require_relative "tokenizers/decoders/bpe_decoder"
+require_relative "tokenizers/decoders/ctc"
+require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/word_piece"
+
+# models
+require_relative "tokenizers/models/bpe"
+require_relative "tokenizers/models/word_level"
+require_relative "tokenizers/models/word_piece"
+require_relative "tokenizers/models/unigram"
+
+# normalizers
+require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/strip"
+
+# pre-tokenizers
+require_relative "tokenizers/pre_tokenizers/byte_level"
+require_relative "tokenizers/pre_tokenizers/digits"
+require_relative "tokenizers/pre_tokenizers/metaspace"
+require_relative "tokenizers/pre_tokenizers/punctuation"
+require_relative "tokenizers/pre_tokenizers/split"
+
+# processors
+require_relative "tokenizers/processors/byte_level"
+require_relative "tokenizers/processors/roberta_processing"
+require_relative "tokenizers/processors/template_processing"
+
+# trainers
+require_relative "tokenizers/trainers/bpe_trainer"
+require_relative "tokenizers/trainers/unigram_trainer"
+require_relative "tokenizers/trainers/word_level_trainer"
+require_relative "tokenizers/trainers/word_piece_trainer"
+
+# other
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"
@@ -15,5 +49,11 @@ require_relative "tokenizers/version"
 module Tokenizers
   class Error < StandardError; end
 
-  extend FromPretrained
+  def self.from_pretrained(...)
+    Tokenizer.from_pretrained(...)
+  end
+
+  def self.from_file(...)
+    Tokenizer.from_file(...)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-22 00:00:00.000000000 Z
+date: 2023-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -45,12 +45,39 @@ files:
 - ext/tokenizers/src/models.rs
 - ext/tokenizers/src/normalizers.rs
 - ext/tokenizers/src/pre_tokenizers.rs
+- ext/tokenizers/src/processors.rs
 - ext/tokenizers/src/tokenizer.rs
+- ext/tokenizers/src/trainers.rs
+- ext/tokenizers/src/utils/mod.rs
+- ext/tokenizers/src/utils/normalization.rs
+- ext/tokenizers/src/utils/regex.rs
 - lib/tokenizers.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/decoders/bpe_decoder.rb
+- lib/tokenizers/decoders/ctc.rb
+- lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/models/bpe.rb
+- lib/tokenizers/models/unigram.rb
+- lib/tokenizers/models/word_level.rb
+- lib/tokenizers/models/word_piece.rb
+- lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/strip.rb
+- lib/tokenizers/pre_tokenizers/byte_level.rb
+- lib/tokenizers/pre_tokenizers/digits.rb
+- lib/tokenizers/pre_tokenizers/metaspace.rb
+- lib/tokenizers/pre_tokenizers/punctuation.rb
+- lib/tokenizers/pre_tokenizers/split.rb
+- lib/tokenizers/processors/byte_level.rb
+- lib/tokenizers/processors/roberta_processing.rb
+- lib/tokenizers/processors/template_processing.rb
 - lib/tokenizers/tokenizer.rb
+- lib/tokenizers/trainers/bpe_trainer.rb
+- lib/tokenizers/trainers/unigram_trainer.rb
+- lib/tokenizers/trainers/word_level_trainer.rb
+- lib/tokenizers/trainers/word_piece_trainer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses:
@@ -71,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby