tokenizers 0.3.2-x86_64-linux-musl

data/lib/tokenizers/tokenizer.rb

@@ -0,0 +1,45 @@
+module Tokenizers
+  class Tokenizer
+    extend FromPretrained
+
+    def to_s(pretty: false)
+      _to_s(pretty)
+    end
+
+    def save(path, pretty: false)
+      _save(path, pretty)
+    end
+
+    def encode(sequence, pair = nil, is_pretokenized: false, add_special_tokens: true)
+      _encode(sequence, pair, is_pretokenized, add_special_tokens)
+    end
+
+    def encode_batch(input, is_pretokenized: false, add_special_tokens: true)
+      _encode_batch(input, is_pretokenized, add_special_tokens)
+    end
+
+    def decode(ids, skip_special_tokens: true)
+      _decode(ids, skip_special_tokens)
+    end
+
+    def decode_batch(sequences, skip_special_tokens: true)
+      _decode_batch(sequences, skip_special_tokens)
+    end
+
+    def enable_padding(**options)
+      _enable_padding(options)
+    end
+
+    def enable_truncation(max_length, **options)
+      _enable_truncation(max_length, options)
+    end
+
+    def vocab(with_added_tokens: true)
+      _vocab(with_added_tokens)
+    end
+
+    def vocab_size(with_added_tokens: true)
+      _vocab_size(with_added_tokens)
+    end
+  end
+end

data/lib/tokenizers/trainers/bpe_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class BpeTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/unigram_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class UnigramTrainer
+      def self.new(vocab_size: 8000,
+                   show_progress: true,
+                   special_tokens: [],
+                   initial_alphabet: [],
+                   shrinking_factor: 0.75,
+                   unk_token: nil,
+                   max_piece_length: 16,
+                   n_sub_iterations: 2)
+
+        _new({
+          vocab_size: vocab_size,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          initial_alphabet: initial_alphabet,
+          shrinking_factor: shrinking_factor,
+          unk_token: unk_token,
+          max_piece_length: max_piece_length,
+          n_sub_iterations: n_sub_iterations
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_level_trainer.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Trainers
+    class WordLevelTrainer
+      def self.new(**options)
+        _new(options)
+      end
+    end
+  end
+end

data/lib/tokenizers/trainers/word_piece_trainer.rb

@@ -0,0 +1,26 @@
+module Tokenizers
+  module Trainers
+    class WordPieceTrainer
+      def self.new(vocab_size: 30000,
+                   min_frequency: 0,
+                   show_progress: true,
+                   special_tokens: [],
+                   limit_alphabet: nil,
+                   initial_alphabet: [],
+                   continuing_subword_prefix: "##",
+                   end_of_word_suffix: nil)
+
+        _new({
+          vocab_size: vocab_size,
+          min_frequency: min_frequency,
+          show_progress: show_progress,
+          special_tokens: special_tokens,
+          limit_alphabet: limit_alphabet,
+          initial_alphabet: initial_alphabet,
+          continuing_subword_prefix: continuing_subword_prefix,
+          end_of_word_suffix: end_of_word_suffix
+        })
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -0,0 +1,3 @@
+module Tokenizers
+  VERSION = "0.3.2"
+end

data/lib/tokenizers.rb

@@ -0,0 +1,59 @@
+# ext
+begin
+  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+rescue LoadError
+  require_relative "tokenizers/tokenizers"
+end
+
+# decoders
+require_relative "tokenizers/decoders/bpe_decoder"
+require_relative "tokenizers/decoders/ctc"
+require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/word_piece"
+
+# models
+require_relative "tokenizers/models/bpe"
+require_relative "tokenizers/models/word_level"
+require_relative "tokenizers/models/word_piece"
+require_relative "tokenizers/models/unigram"
+
+# normalizers
+require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/strip"
+
+# pre-tokenizers
+require_relative "tokenizers/pre_tokenizers/byte_level"
+require_relative "tokenizers/pre_tokenizers/digits"
+require_relative "tokenizers/pre_tokenizers/metaspace"
+require_relative "tokenizers/pre_tokenizers/punctuation"
+require_relative "tokenizers/pre_tokenizers/split"
+
+# processors
+require_relative "tokenizers/processors/byte_level"
+require_relative "tokenizers/processors/roberta_processing"
+require_relative "tokenizers/processors/template_processing"
+
+# trainers
+require_relative "tokenizers/trainers/bpe_trainer"
+require_relative "tokenizers/trainers/unigram_trainer"
+require_relative "tokenizers/trainers/word_level_trainer"
+require_relative "tokenizers/trainers/word_piece_trainer"
+
+# other
+require_relative "tokenizers/char_bpe_tokenizer"
+require_relative "tokenizers/encoding"
+require_relative "tokenizers/from_pretrained"
+require_relative "tokenizers/tokenizer"
+require_relative "tokenizers/version"
+
+module Tokenizers
+  class Error < StandardError; end
+
+  def self.from_pretrained(...)
+    Tokenizer.from_pretrained(...)
+  end
+
+  def self.from_file(...)
+    Tokenizer.from_file(...)
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: tokenizers
+version: !ruby/object:Gem::Version
+  version: 0.3.2
+platform: x86_64-linux-musl
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-07 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: andrew@ankane.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE-THIRD-PARTY.txt
+- LICENSE.txt
+- README.md
+- lib/tokenizers.rb
+- lib/tokenizers/2.7/tokenizers.so
+- lib/tokenizers/3.0/tokenizers.so
+- lib/tokenizers/3.1/tokenizers.so
+- lib/tokenizers/3.2/tokenizers.so
+- lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/decoders/bpe_decoder.rb
+- lib/tokenizers/decoders/ctc.rb
+- lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/word_piece.rb
+- lib/tokenizers/encoding.rb
+- lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/models/bpe.rb
+- lib/tokenizers/models/unigram.rb
+- lib/tokenizers/models/word_level.rb
+- lib/tokenizers/models/word_piece.rb
+- lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/strip.rb
+- lib/tokenizers/pre_tokenizers/byte_level.rb
+- lib/tokenizers/pre_tokenizers/digits.rb
+- lib/tokenizers/pre_tokenizers/metaspace.rb
+- lib/tokenizers/pre_tokenizers/punctuation.rb
+- lib/tokenizers/pre_tokenizers/split.rb
+- lib/tokenizers/processors/byte_level.rb
+- lib/tokenizers/processors/roberta_processing.rb
+- lib/tokenizers/processors/template_processing.rb
+- lib/tokenizers/tokenizer.rb
+- lib/tokenizers/trainers/bpe_trainer.rb
+- lib/tokenizers/trainers/unigram_trainer.rb
+- lib/tokenizers/trainers/word_level_trainer.rb
+- lib/tokenizers/trainers/word_piece_trainer.rb
+- lib/tokenizers/version.rb
+homepage: https://github.com/ankane/tokenizers-ruby
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Fast state-of-the-art tokenizers for Ruby
+test_files: []