transformers-rb 0.1.0

data/lib/transformers/models/bert/tokenization_bert.rb

@@ -0,0 +1,115 @@
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Bert
+    class BertTokenizer < PreTrainedTokenizer
+      class BasicTokenizer
+        attr_reader :do_lower_case, :tokenize_chinese_chars, :never_split, :strip_accents, :do_split_on_punc
+
+        def initialize(
+          do_lower_case: true,
+          never_split: nil,
+          tokenize_chinese_chars: true,
+          strip_accents: nil,
+          do_split_on_punc: true
+        )
+          if never_split.nil?
+            never_split = []
+          end
+          @do_lower_case = do_lower_case
+          @never_split = Set.new(never_split)
+          @tokenize_chinese_chars = tokenize_chinese_chars
+          @strip_accents = strip_accents
+          @do_split_on_punc = do_split_on_punc
+        end
+      end
+
+      class WordpieceTokenizer
+        def initialize(vocab:, unk_token:, max_input_chars_per_word: 100)
+          @vocab = vocab
+          @unk_token = unk_token
+          @max_input_chars_per_word = max_input_chars_per_word
+        end
+      end
+
+      attr_reader :vocab, :basic_tokenizer
+
+      def initialize(
+        vocab_file:,
+        do_lower_case: true,
+        do_basic_tokenize: true,
+        never_split: nil,
+        unk_token: "[UNK]",
+        sep_token: "[SEP]",
+        pad_token: "[PAD]",
+        cls_token: "[CLS]",
+        mask_token: "[MASK]",
+        tokenize_chinese_chars: true,
+        strip_accents: nil,
+        **kwargs
+      )
+        if !File.exist?(vocab_file)
+          raise ArgumentError,
+            "Can't find a vocabulary file at path '#{vocab_file}'. To load the vocabulary from a Google pretrained" +
+            " model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+        end
+        @vocab = load_vocab(vocab_file)
+        @ids_to_tokens = @vocab.invert
+        @do_basic_tokenize = do_basic_tokenize
+        if do_basic_tokenize
+          @basic_tokenizer =
+            BasicTokenizer.new(
+              do_lower_case: do_lower_case,
+              never_split: never_split,
+              tokenize_chinese_chars: tokenize_chinese_chars,
+              strip_accents: strip_accents
+            )
+        end
+
+        @wordpiece_tokenizer = WordpieceTokenizer.new(vocab: @vocab, unk_token: unk_token.to_s)
+
+        super(
+          do_lower_case: do_lower_case,
+          do_basic_tokenize: do_basic_tokenize,
+          never_split: never_split,
+          unk_token: unk_token,
+          sep_token: sep_token,
+          pad_token: pad_token,
+          cls_token: cls_token,
+          mask_token: mask_token,
+          tokenize_chinese_chars: tokenize_chinese_chars,
+          strip_accents: strip_accents,
+          **kwargs
+        )
+      end
+
+      def _convert_token_to_id(token)
+        @vocab.fetch(token, @vocab.fetch(@unk_token))
+      end
+
+      private
+
+      def load_vocab(vocab_file)
+        vocab = {}
+        tokens = File.readlines(vocab_file)
+        tokens.each_with_index do |token, index|
+          token = token.chomp
+          vocab[token] = index
+        end
+        vocab
+      end
+    end
+  end
+end

data/lib/transformers/models/bert/tokenization_bert_fast.rb

@@ -0,0 +1,52 @@
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Bert
+    class BertTokenizerFast < PreTrainedTokenizerFast
+      VOCAB_FILES_NAMES = {vocab_file: "vocab.txt", tokenizer_file: "tokenizer.json"}
+
+      self.vocab_files_names = VOCAB_FILES_NAMES
+      self.slow_tokenizer_class = BertTokenizer
+
+      def initialize(
+        vocab_file: nil,
+        tokenizer_file: nil,
+        do_lower_case: true,
+        unk_token: "[UNK]",
+        sep_token: "[SEP]",
+        pad_token: "[PAD]",
+        cls_token: "[CLS]",
+        mask_token: "[MASK]",
+        tokenize_chinese_chars: true,
+        strip_accents: nil,
+        **kwargs
+      )
+        super(
+          vocab_file,
+          tokenizer_file: tokenizer_file,
+          do_lower_case: do_lower_case,
+          unk_token: unk_token,
+          sep_token: sep_token,
+          pad_token: pad_token,
+          cls_token: cls_token,
+          mask_token: mask_token,
+          tokenize_chinese_chars: tokenize_chinese_chars,
+          strip_accents: strip_accents,
+          **kwargs
+        )
+      end
+    end
+  end
+end

data/lib/transformers/models/distilbert/configuration_distilbert.rb

@@ -0,0 +1,63 @@
+# Copyright 2019-present, the HuggingFace Inc. team, The Google AI Language Team and Facebook, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Distilbert
+    class DistilBertConfig < PretrainedConfig
+      self.model_type = "distilbert"
+      self.attribute_map = {
+        hidden_size: "dim",
+        num_attention_heads: "n_heads",
+        num_hidden_layers: "n_layers",
+      }
+
+     attr_reader :vocab_size, :max_position_embeddings, :sinusoidal_pos_embds, :n_layers, :n_heads,
+      :dim, :hidden_dim, :dropout, :attention_dropout, :activation, :initializer_range, :qa_dropout,
+      :seq_classif_dropout, :pad_token_id
+
+      def initialize(
+        vocab_size: 30522,
+        max_position_embeddings: 512,
+        sinusoidal_pos_embds: false,
+        n_layers: 6,
+        n_heads: 12,
+        dim: 768,
+        hidden_dim: 4 * 768,
+        dropout: 0.1,
+        attention_dropout: 0.1,
+        activation: "gelu",
+        initializer_range: 0.02,
+        qa_dropout: 0.1,
+        seq_classif_dropout: 0.2,
+        pad_token_id: 0,
+        **kwargs
+      )
+        @vocab_size = vocab_size
+        @max_position_embeddings = max_position_embeddings
+        @sinusoidal_pos_embds = sinusoidal_pos_embds
+        @n_layers = n_layers
+        @n_heads = n_heads
+        @dim = dim
+        @hidden_dim = hidden_dim
+        @dropout = dropout
+        @attention_dropout = attention_dropout
+        @activation = activation
+        @initializer_range = initializer_range
+        @qa_dropout = qa_dropout
+        @seq_classif_dropout = seq_classif_dropout
+        super(**kwargs, pad_token_id: pad_token_id)
+      end
+    end
+  end
+end