sorbet-baml 0.1.0 → 0.3.0

data/sorbet/rbi/gems/tokenizers@0.5.5.rbi

@@ -0,0 +1,754 @@
+# typed: true
+
+# DO NOT EDIT MANUALLY
+# This is an autogenerated file for types exported from the `tokenizers` gem.
+# Please instead update this file by running `bin/tapioca gem tokenizers`.
+
+
+# source://tokenizers//lib/tokenizers/decoders/bpe_decoder.rb#1
+module Tokenizers
+  class << self
+    # source://tokenizers//lib/tokenizers.rb#59
+    def from_file(*_arg0, **_arg1, &_arg2); end
+
+    # source://tokenizers//lib/tokenizers.rb#55
+    def from_pretrained(*_arg0, **_arg1, &_arg2); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/added_token.rb#2
+class Tokenizers::AddedToken
+  def content; end
+  def lstrip; end
+  def normalized; end
+  def rstrip; end
+  def single_word; end
+  def special; end
+
+  class << self
+    def _new(_arg0, _arg1); end
+
+    # source://tokenizers//lib/tokenizers/added_token.rb#3
+    def new(content, **kwargs); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/char_bpe_tokenizer.rb#2
+class Tokenizers::CharBPETokenizer
+  # @return [CharBPETokenizer] a new instance of CharBPETokenizer
+  #
+  # source://tokenizers//lib/tokenizers/char_bpe_tokenizer.rb#3
+  def initialize(vocab, merges, unk_token: T.unsafe(nil), suffix: T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/char_bpe_tokenizer.rb#18
+  def decode(ids); end
+
+  # source://tokenizers//lib/tokenizers/char_bpe_tokenizer.rb#14
+  def encode(text, **options); end
+end
+
+# source://tokenizers//lib/tokenizers/decoders/bpe_decoder.rb#2
+module Tokenizers::Decoders; end
+
+# source://tokenizers//lib/tokenizers/decoders/bpe_decoder.rb#3
+class Tokenizers::Decoders::BPEDecoder < ::Tokenizers::Decoders::Decoder
+  def suffix; end
+  def suffix=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/decoders/bpe_decoder.rb#4
+    def new(suffix: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Decoders::ByteFallback < ::Tokenizers::Decoders::Decoder
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Decoders::ByteLevel < ::Tokenizers::Decoders::Decoder
+  class << self
+    def new; end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/decoders/ctc.rb#3
+class Tokenizers::Decoders::CTC < ::Tokenizers::Decoders::Decoder
+  def cleanup; end
+  def cleanup=(_arg0); end
+  def pad_token; end
+  def pad_token=(_arg0); end
+  def word_delimiter_token; end
+  def word_delimiter_token=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/decoders/ctc.rb#4
+    def new(pad_token: T.unsafe(nil), word_delimiter_token: T.unsafe(nil), cleanup: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Decoders::Decoder
+  def decode(_arg0); end
+end
+
+class Tokenizers::Decoders::Fuse < ::Tokenizers::Decoders::Decoder
+  class << self
+    def new; end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/decoders/metaspace.rb#3
+class Tokenizers::Decoders::Metaspace < ::Tokenizers::Decoders::Decoder
+  def prepend_scheme; end
+  def prepend_scheme=(_arg0); end
+  def replacement; end
+  def replacement=(_arg0); end
+  def split; end
+  def split=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/decoders/metaspace.rb#4
+    def new(replacement: T.unsafe(nil), prepend_scheme: T.unsafe(nil), split: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Decoders::Replace < ::Tokenizers::Decoders::Decoder
+  class << self
+    def new(_arg0, _arg1); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/decoders/strip.rb#3
+class Tokenizers::Decoders::Strip < ::Tokenizers::Decoders::Decoder
+  def content; end
+  def content=(_arg0); end
+  def start; end
+  def start=(_arg0); end
+  def stop; end
+  def stop=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/decoders/strip.rb#4
+    def new(content: T.unsafe(nil), start: T.unsafe(nil), stop: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/decoders/word_piece.rb#3
+class Tokenizers::Decoders::WordPiece < ::Tokenizers::Decoders::Decoder
+  def cleanup; end
+  def cleanup=(_arg0); end
+  def prefix; end
+  def prefix=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1); end
+
+    # source://tokenizers//lib/tokenizers/decoders/word_piece.rb#4
+    def new(prefix: T.unsafe(nil), cleanup: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/encoding.rb#2
+class Tokenizers::Encoding
+  def _char_to_token(_arg0, _arg1); end
+  def _char_to_word(_arg0, _arg1); end
+  def _word_to_chars(_arg0, _arg1); end
+  def _word_to_tokens(_arg0, _arg1); end
+  def attention_mask; end
+
+  # source://tokenizers//lib/tokenizers/encoding.rb#11
+  def char_to_token(char_pos, sequence_index = T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/encoding.rb#15
+  def char_to_word(char_pos, sequence_index = T.unsafe(nil)); end
+
+  def ids; end
+  def n_sequences; end
+  def offsets; end
+  def overflowing; end
+  def sequence_ids; end
+  def special_tokens_mask; end
+  def token_to_chars(_arg0); end
+  def token_to_sequence(_arg0); end
+  def token_to_word(_arg0); end
+  def tokens; end
+  def type_ids; end
+  def word_ids; end
+
+  # source://tokenizers//lib/tokenizers/encoding.rb#7
+  def word_to_chars(word_index, sequence_index = T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/encoding.rb#3
+  def word_to_tokens(word_index, sequence_index = T.unsafe(nil)); end
+end
+
+# source://tokenizers//lib/tokenizers.rb#53
+class Tokenizers::Error < ::StandardError; end
+
+# source://tokenizers//lib/tokenizers/from_pretrained.rb#2
+module Tokenizers::FromPretrained
+  # use Ruby for downloads
+  # this avoids the need to vendor OpenSSL on Linux
+  # and reduces the extension size by about half
+  #
+  # source://tokenizers//lib/tokenizers/from_pretrained.rb#9
+  def from_pretrained(identifier, revision: T.unsafe(nil), auth_token: T.unsafe(nil)); end
+
+  private
+
+  # source://tokenizers//lib/tokenizers/from_pretrained.rb#96
+  def cache_dir; end
+
+  # use same storage format as Rust version
+  # https://github.com/epwalsh/rust-cached-path
+  #
+  # source://tokenizers//lib/tokenizers/from_pretrained.rb#46
+  def cached_path(cache_dir, url, headers, options); end
+
+  # source://tokenizers//lib/tokenizers/from_pretrained.rb#115
+  def ensure_cache_dir; end
+
+  # @return [Boolean]
+  #
+  # source://tokenizers//lib/tokenizers/from_pretrained.rb#121
+  def mac?; end
+end
+
+# for user agent
+#
+# source://tokenizers//lib/tokenizers/from_pretrained.rb#4
+Tokenizers::FromPretrained::TOKENIZERS_VERSION = T.let(T.unsafe(nil), String)
+
+# source://tokenizers//lib/tokenizers/models/bpe.rb#2
+module Tokenizers::Models; end
+
+# source://tokenizers//lib/tokenizers/models/bpe.rb#3
+class Tokenizers::Models::BPE < ::Tokenizers::Models::Model
+  def byte_fallback; end
+  def byte_fallback=(_arg0); end
+  def continuing_subword_prefix; end
+  def continuing_subword_prefix=(_arg0); end
+  def dropout; end
+  def dropout=(_arg0); end
+  def end_of_word_suffix; end
+  def end_of_word_suffix=(_arg0); end
+  def fuse_unk; end
+  def fuse_unk=(_arg0); end
+  def unk_token; end
+  def unk_token=(_arg0); end
+
+  class << self
+    def _from_file(_arg0, _arg1, _arg2); end
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/models/bpe.rb#4
+    def new(vocab: T.unsafe(nil), merges: T.unsafe(nil), **kwargs); end
+  end
+end
+
+class Tokenizers::Models::Model; end
+
+# source://tokenizers//lib/tokenizers/models/unigram.rb#3
+class Tokenizers::Models::Unigram < ::Tokenizers::Models::Model
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/models/unigram.rb#4
+    def new(vocab: T.unsafe(nil), unk_id: T.unsafe(nil), byte_fallback: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/models/word_level.rb#3
+class Tokenizers::Models::WordLevel < ::Tokenizers::Models::Model
+  def unk_token; end
+  def unk_token=(_arg0); end
+
+  class << self
+    def _from_file(_arg0, _arg1); end
+    def _new(_arg0, _arg1); end
+
+    # source://tokenizers//lib/tokenizers/models/word_level.rb#8
+    def from_file(vocab, unk_token: T.unsafe(nil)); end
+
+    # source://tokenizers//lib/tokenizers/models/word_level.rb#4
+    def new(vocab: T.unsafe(nil), unk_token: T.unsafe(nil)); end
+
+    def read_file(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/models/word_piece.rb#3
+class Tokenizers::Models::WordPiece < ::Tokenizers::Models::Model
+  def continuing_subword_prefix; end
+  def continuing_subword_prefix=(_arg0); end
+  def max_input_chars_per_word; end
+  def max_input_chars_per_word=(_arg0); end
+  def unk_token; end
+  def unk_token=(_arg0); end
+
+  class << self
+    def _from_file(_arg0, _arg1); end
+    def _new(_arg0, _arg1); end
+
+    # source://tokenizers//lib/tokenizers/models/word_piece.rb#4
+    def new(vocab: T.unsafe(nil), **kwargs); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/normalizers/bert_normalizer.rb#2
+module Tokenizers::Normalizers; end
+
+# source://tokenizers//lib/tokenizers/normalizers/bert_normalizer.rb#3
+class Tokenizers::Normalizers::BertNormalizer < ::Tokenizers::Normalizers::Normalizer
+  def clean_text; end
+  def clean_text=(_arg0); end
+  def handle_chinese_chars; end
+  def handle_chinese_chars=(_arg0); end
+  def lowercase; end
+  def lowercase=(_arg0); end
+  def strip_accents; end
+  def strip_accents=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1, _arg2, _arg3); end
+
+    # source://tokenizers//lib/tokenizers/normalizers/bert_normalizer.rb#4
+    def new(clean_text: T.unsafe(nil), handle_chinese_chars: T.unsafe(nil), strip_accents: T.unsafe(nil), lowercase: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Normalizers::Lowercase < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::NFC < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::NFD < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::NFKC < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::NFKD < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::Nmt < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::Normalizers::Normalizer
+  def normalize_str(_arg0); end
+end
+
+class Tokenizers::Normalizers::Precompiled < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/normalizers/prepend.rb#3
+class Tokenizers::Normalizers::Prepend < ::Tokenizers::Normalizers::Normalizer
+  def prepend; end
+  def prepend=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/normalizers/prepend.rb#4
+    def new(prepend: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Normalizers::Replace < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new(_arg0, _arg1); end
+  end
+end
+
+class Tokenizers::Normalizers::Sequence < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/normalizers/strip.rb#3
+class Tokenizers::Normalizers::Strip < ::Tokenizers::Normalizers::Normalizer
+  def left; end
+  def left=(_arg0); end
+  def right; end
+  def right=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1); end
+
+    # source://tokenizers//lib/tokenizers/normalizers/strip.rb#4
+    def new(left: T.unsafe(nil), right: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Normalizers::StripAccents < ::Tokenizers::Normalizers::Normalizer
+  class << self
+    def new; end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/byte_level.rb#2
+module Tokenizers::PreTokenizers; end
+
+class Tokenizers::PreTokenizers::BertPreTokenizer < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def new; end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/byte_level.rb#3
+class Tokenizers::PreTokenizers::ByteLevel < ::Tokenizers::PreTokenizers::PreTokenizer
+  def add_prefix_space; end
+  def add_prefix_space=(_arg0); end
+  def use_regex; end
+  def use_regex=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1); end
+    def alphabet; end
+
+    # source://tokenizers//lib/tokenizers/pre_tokenizers/byte_level.rb#4
+    def new(add_prefix_space: T.unsafe(nil), use_regex: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::PreTokenizers::CharDelimiterSplit < ::Tokenizers::PreTokenizers::PreTokenizer
+  def delimiter; end
+  def delimiter=(_arg0); end
+
+  class << self
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/digits.rb#3
+class Tokenizers::PreTokenizers::Digits < ::Tokenizers::PreTokenizers::PreTokenizer
+  def individual_digits; end
+  def individual_digits=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/pre_tokenizers/digits.rb#4
+    def new(individual_digits: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/metaspace.rb#3
+class Tokenizers::PreTokenizers::Metaspace < ::Tokenizers::PreTokenizers::PreTokenizer
+  def prepend_scheme; end
+  def prepend_scheme=(_arg0); end
+  def replacement; end
+  def replacement=(_arg0); end
+  def split; end
+  def split=(_arg0); end
+
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/pre_tokenizers/metaspace.rb#4
+    def new(replacement: T.unsafe(nil), prepend_scheme: T.unsafe(nil), split: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::PreTokenizers::PreTokenizer
+  def pre_tokenize_str(_arg0); end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/punctuation.rb#3
+class Tokenizers::PreTokenizers::Punctuation < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/pre_tokenizers/punctuation.rb#4
+    def new(behavior: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::PreTokenizers::Sequence < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/pre_tokenizers/split.rb#3
+class Tokenizers::PreTokenizers::Split < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/pre_tokenizers/split.rb#4
+    def new(pattern, behavior, invert: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::PreTokenizers::UnicodeScripts < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::PreTokenizers::Whitespace < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def new; end
+  end
+end
+
+class Tokenizers::PreTokenizers::WhitespaceSplit < ::Tokenizers::PreTokenizers::PreTokenizer
+  class << self
+    def new; end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/processors/byte_level.rb#2
+module Tokenizers::Processors; end
+
+class Tokenizers::Processors::BertProcessing < ::Tokenizers::Processors::PostProcessor
+  class << self
+    def new(_arg0, _arg1); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/processors/byte_level.rb#3
+class Tokenizers::Processors::ByteLevel < ::Tokenizers::Processors::PostProcessor
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/processors/byte_level.rb#4
+    def new(trim_offsets: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Processors::PostProcessor; end
+
+# source://tokenizers//lib/tokenizers/processors/roberta_processing.rb#3
+class Tokenizers::Processors::RobertaProcessing < ::Tokenizers::Processors::PostProcessor
+  class << self
+    def _new(_arg0, _arg1, _arg2, _arg3); end
+
+    # source://tokenizers//lib/tokenizers/processors/roberta_processing.rb#4
+    def new(sep, cls, trim_offsets: T.unsafe(nil), add_prefix_space: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/processors/template_processing.rb#3
+class Tokenizers::Processors::TemplateProcessing < ::Tokenizers::Processors::PostProcessor
+  class << self
+    def _new(_arg0, _arg1, _arg2); end
+
+    # source://tokenizers//lib/tokenizers/processors/template_processing.rb#4
+    def new(single: T.unsafe(nil), pair: T.unsafe(nil), special_tokens: T.unsafe(nil)); end
+  end
+end
+
+class Tokenizers::Regex
+  class << self
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/tokenizer.rb#2
+class Tokenizers::Tokenizer
+  extend ::Tokenizers::FromPretrained
+
+  def _decode(_arg0, _arg1); end
+  def _decode_batch(_arg0, _arg1); end
+  def _enable_padding(_arg0); end
+  def _enable_truncation(_arg0, _arg1); end
+  def _encode(_arg0, _arg1, _arg2, _arg3); end
+  def _encode_batch(_arg0, _arg1, _arg2); end
+  def _save(_arg0, _arg1); end
+  def _to_s(_arg0); end
+  def _vocab(_arg0); end
+  def _vocab_size(_arg0); end
+  def add_special_tokens(_arg0); end
+  def add_tokens(_arg0); end
+  def added_tokens_decoder; end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#21
+  def decode(ids, skip_special_tokens: T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#25
+  def decode_batch(sequences, skip_special_tokens: T.unsafe(nil)); end
+
+  def decoder; end
+  def decoder=(_arg0); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#29
+  def enable_padding(**options); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#33
+  def enable_truncation(max_length, **options); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#13
+  def encode(sequence, pair = T.unsafe(nil), is_pretokenized: T.unsafe(nil), add_special_tokens: T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#17
+  def encode_batch(input, is_pretokenized: T.unsafe(nil), add_special_tokens: T.unsafe(nil)); end
+
+  def id_to_token(_arg0); end
+  def model; end
+  def model=(_arg0); end
+  def no_padding; end
+  def no_truncation; end
+  def normalizer; end
+  def normalizer=(_arg0); end
+  def num_special_tokens_to_add(_arg0); end
+  def padding; end
+  def post_processor; end
+  def post_processor=(_arg0); end
+  def pre_tokenizer; end
+  def pre_tokenizer=(_arg0); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#9
+  def save(path, pretty: T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#5
+  def to_s(pretty: T.unsafe(nil)); end
+
+  def token_to_id(_arg0); end
+  def train(_arg0, _arg1); end
+  def truncation; end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#37
+  def vocab(with_added_tokens: T.unsafe(nil)); end
+
+  # source://tokenizers//lib/tokenizers/tokenizer.rb#41
+  def vocab_size(with_added_tokens: T.unsafe(nil)); end
+
+  class << self
+    def from_file(_arg0); end
+    def from_str(_arg0); end
+    def new(_arg0); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/trainers/bpe_trainer.rb#2
+module Tokenizers::Trainers; end
+
+# source://tokenizers//lib/tokenizers/trainers/bpe_trainer.rb#3
+class Tokenizers::Trainers::BpeTrainer < ::Tokenizers::Trainers::Trainer
+  def continuing_subword_prefix; end
+  def continuing_subword_prefix=(_arg0); end
+  def end_of_word_suffix; end
+  def end_of_word_suffix=(_arg0); end
+  def initial_alphabet; end
+  def initial_alphabet=(_arg0); end
+  def limit_alphabet; end
+  def limit_alphabet=(_arg0); end
+  def min_frequency; end
+  def min_frequency=(_arg0); end
+  def show_progress; end
+  def show_progress=(_arg0); end
+  def special_tokens; end
+  def special_tokens=(_arg0); end
+  def vocab_size; end
+  def vocab_size=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/trainers/bpe_trainer.rb#4
+    def new(**options); end
+  end
+end
+
+class Tokenizers::Trainers::Trainer; end
+
+# source://tokenizers//lib/tokenizers/trainers/unigram_trainer.rb#3
+class Tokenizers::Trainers::UnigramTrainer < ::Tokenizers::Trainers::Trainer
+  def initial_alphabet; end
+  def initial_alphabet=(_arg0); end
+  def show_progress; end
+  def show_progress=(_arg0); end
+  def special_tokens; end
+  def special_tokens=(_arg0); end
+  def vocab_size; end
+  def vocab_size=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/trainers/unigram_trainer.rb#4
+    def new(vocab_size: T.unsafe(nil), show_progress: T.unsafe(nil), special_tokens: T.unsafe(nil), initial_alphabet: T.unsafe(nil), shrinking_factor: T.unsafe(nil), unk_token: T.unsafe(nil), max_piece_length: T.unsafe(nil), n_sub_iterations: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/trainers/word_level_trainer.rb#3
+class Tokenizers::Trainers::WordLevelTrainer < ::Tokenizers::Trainers::Trainer
+  def min_frequency; end
+  def min_frequency=(_arg0); end
+  def show_progress; end
+  def show_progress=(_arg0); end
+  def special_tokens; end
+  def special_tokens=(_arg0); end
+  def vocab_size; end
+  def vocab_size=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/trainers/word_level_trainer.rb#4
+    def new(**options); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/trainers/word_piece_trainer.rb#3
+class Tokenizers::Trainers::WordPieceTrainer < ::Tokenizers::Trainers::Trainer
+  def continuing_subword_prefix; end
+  def continuing_subword_prefix=(_arg0); end
+  def end_of_word_suffix; end
+  def end_of_word_suffix=(_arg0); end
+  def initial_alphabet; end
+  def initial_alphabet=(_arg0); end
+  def limit_alphabet; end
+  def limit_alphabet=(_arg0); end
+  def min_frequency; end
+  def min_frequency=(_arg0); end
+  def show_progress; end
+  def show_progress=(_arg0); end
+  def special_tokens; end
+  def special_tokens=(_arg0); end
+  def vocab_size; end
+  def vocab_size=(_arg0); end
+
+  class << self
+    def _new(_arg0); end
+
+    # source://tokenizers//lib/tokenizers/trainers/word_piece_trainer.rb#4
+    def new(vocab_size: T.unsafe(nil), min_frequency: T.unsafe(nil), show_progress: T.unsafe(nil), special_tokens: T.unsafe(nil), limit_alphabet: T.unsafe(nil), initial_alphabet: T.unsafe(nil), continuing_subword_prefix: T.unsafe(nil), end_of_word_suffix: T.unsafe(nil)); end
+  end
+end
+
+# source://tokenizers//lib/tokenizers/version.rb#2
+Tokenizers::VERSION = T.let(T.unsafe(nil), String)