transformers-rb 0.1.0

data/lib/transformers/models/distilbert/tokenization_distilbert.rb

@@ -0,0 +1,114 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Distilbert
+    class DistilBertTokenizer < PreTrainedTokenizer
+      VOCAB_FILES_NAMES = {vocab_file: "vocab.txt"}
+
+      self.vocab_files_names = VOCAB_FILES_NAMES
+      self.model_input_names = ["input_ids", "attention_mask"]
+
+      class BasicTokenizer
+        attr_reader :do_lower_case, :tokenize_chinese_chars, :strip_accents
+
+        def initialize(
+          do_lower_case: true,
+          never_split: nil,
+          tokenize_chinese_chars: true,
+          strip_accents: nil,
+          do_split_on_punc: true
+        )
+          if never_split.nil?
+            never_split = []
+          end
+          @do_lower_case = do_lower_case
+          @never_split = Set.new(never_split)
+          @tokenize_chinese_chars = tokenize_chinese_chars
+          @strip_accents = strip_accents
+          @do_split_on_punc = do_split_on_punc
+        end
+      end
+
+      class WordpieceTokenizer
+        def initialize(vocab:, unk_token:, max_input_chars_per_word: 100)
+          @vocab = vocab
+          @unk_token = unk_token
+          @max_input_chars_per_word = max_input_chars_per_word
+        end
+      end
+
+      attr_reader :vocab, :basic_tokenizer
+
+      def initialize(
+        vocab_file:,
+        do_lower_case: true,
+        do_basic_tokenize: true,
+        never_split: nil,
+        unk_token: "[UNK]",
+        sep_token: "[SEP]",
+        pad_token: "[PAD]",
+        cls_token: "[CLS]",
+        mask_token: "[MASK]",
+        tokenize_chinese_chars: true,
+        strip_accents: nil,
+        **kwargs
+      )
+        @vocab = load_vocab(vocab_file)
+        @ids_to_tokens = @vocab.invert
+        @do_basic_tokenize = do_basic_tokenize
+        if do_basic_tokenize
+          @basic_tokenizer =
+            BasicTokenizer.new(
+              do_lower_case: do_lower_case,
+              never_split: never_split,
+              tokenize_chinese_chars: tokenize_chinese_chars,
+              strip_accents: strip_accents
+            )
+        end
+        @wordpiece_tokenizer = WordpieceTokenizer.new(vocab: @vocab, unk_token: unk_token.to_s)
+
+        super(
+          do_lower_case: do_lower_case,
+          do_basic_tokenize: do_basic_tokenize,
+          never_split: never_split,
+          unk_token: unk_token,
+          sep_token: sep_token,
+          pad_token: pad_token,
+          cls_token: cls_token,
+          mask_token: mask_token,
+          tokenize_chinese_chars: tokenize_chinese_chars,
+          strip_accents: strip_accents,
+          **kwargs
+        )
+      end
+
+      def _convert_token_to_id(token)
+        @vocab.fetch(token, @vocab.fetch(@unk_token))
+      end
+
+      private
+
+      def load_vocab(vocab_file)
+        vocab = {}
+        tokens = File.readlines(vocab_file)
+        tokens.each_with_index do |token, index|
+          token = token.chomp
+          vocab[token] = index
+        end
+        vocab
+      end
+    end
+  end
+end

data/lib/transformers/models/distilbert/tokenization_distilbert_fast.rb

@@ -0,0 +1,71 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Distilbert
+    class DistilBertTokenizerFast < PreTrainedTokenizerFast
+      VOCAB_FILES_NAMES = {vocab_file: "vocab.txt", tokenizer_file: "tokenizer.json"}
+
+      self.vocab_files_names = VOCAB_FILES_NAMES
+      self.model_input_names = ["input_ids", "attention_mask"]
+      self.slow_tokenizer_class = DistilBertTokenizer
+
+      def initialize(
+        vocab_file: nil,
+        tokenizer_file: nil,
+        do_lower_case: true,
+        unk_token: "[UNK]",
+        sep_token: "[SEP]",
+        pad_token: "[PAD]",
+        cls_token: "[CLS]",
+        mask_token: "[MASK]",
+        tokenize_chinese_chars: true,
+        strip_accents: nil,
+        **kwargs
+      )
+        super(
+          vocab_file,
+          tokenizer_file: tokenizer_file,
+          do_lower_case: do_lower_case,
+          unk_token: unk_token,
+          sep_token: sep_token,
+          pad_token: pad_token,
+          cls_token: cls_token,
+          mask_token: mask_token,
+          tokenize_chinese_chars: tokenize_chinese_chars,
+          strip_accents: strip_accents,
+          **kwargs
+        )
+
+        if @backend_tokenizer
+          raise Todo
+        end
+
+        @do_lower_case = do_lower_case
+      end
+
+      def build_inputs_with_special_tokens(token_ids_0, token_ids_1 = nil)
+        raise Todo
+      end
+
+      def create_token_type_ids_from_sequences(token_ids_0, token_ids_1 = nil)
+        raise Todo
+      end
+
+      def save_vocabulary(save_directory, filename_prefix: nil)
+        raise Todo
+      end
+    end
+  end
+end

data/lib/transformers/models/vit/configuration_vit.rb

@@ -0,0 +1,60 @@
+# Copyright 2021 Google AI and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Vit
+    class ViTConfig < PretrainedConfig
+      self.model_type = "vit"
+
+      attr_reader :hidden_size, :num_hidden_layers, :num_attention_heads, :intermediate_size,
+        :hidden_act, :hidden_dropout_prob, :attention_probs_dropout_prob, :initializer_range,
+        :layer_norm_eps, :image_size, :patch_size, :num_channels, :qkv_bias, :encoder_stride
+
+      def initialize(
+        hidden_size: 768,
+        num_hidden_layers: 12,
+        num_attention_heads: 12,
+        intermediate_size: 3072,
+        hidden_act: "gelu",
+        hidden_dropout_prob: 0.0,
+        attention_probs_dropout_prob: 0.0,
+        initializer_range: 0.02,
+        layer_norm_eps: 1e-12,
+        image_size: 224,
+        patch_size: 16,
+        num_channels: 3,
+        qkv_bias: true,
+        encoder_stride: 16,
+        **kwargs
+      )
+        super(**kwargs)
+
+        @hidden_size = hidden_size
+        @num_hidden_layers = num_hidden_layers
+        @num_attention_heads = num_attention_heads
+        @intermediate_size = intermediate_size
+        @hidden_act = hidden_act
+        @hidden_dropout_prob = hidden_dropout_prob
+        @attention_probs_dropout_prob = attention_probs_dropout_prob
+        @initializer_range = initializer_range
+        @layer_norm_eps = layer_norm_eps
+        @image_size = image_size
+        @patch_size = patch_size
+        @num_channels = num_channels
+        @qkv_bias = qkv_bias
+        @encoder_stride = encoder_stride
+      end
+    end
+  end
+end

data/lib/transformers/models/vit/image_processing_vit.rb

@@ -0,0 +1,170 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Vit
+    class ViTImageProcessor < BaseImageProcessor
+      def initialize(
+        do_resize: true,
+        size: nil,
+        resample: :bilinear,
+        do_rescale: true,
+        rescale_factor: 1 / 255.0,
+        do_normalize: true,
+        image_mean: nil,
+        image_std: nil,
+        **kwargs
+      )
+        super(**kwargs)
+        size = !size.nil? ? size : {height: 224, width: 224}
+        size = ImageProcessingUtils.get_size_dict(size)
+        @do_resize = do_resize
+        @do_rescale = do_rescale
+        @do_normalize = do_normalize
+        @size = size
+        @resample = resample
+        @rescale_factor = rescale_factor
+        @image_mean = !image_mean.nil? ? image_mean : IMAGENET_STANDARD_MEAN
+        @image_std = !image_std.nil? ? image_std : IMAGENET_STANDARD_STD
+        @valid_processor_keys = [
+          :images,
+          :do_resize,
+          :size,
+          :resample,
+          :do_rescale,
+          :rescale_factor,
+          :do_normalize,
+          :image_mean,
+          :image_std,
+          :return_tensors,
+          :data_format,
+          :input_data_format
+        ]
+      end
+
+      def resize(
+        image,
+        size,
+        resample: :bilinear,
+        data_format: nil,
+        input_data_format: nil,
+        **kwargs
+      )
+        size = ImageProcessingUtils.get_size_dict(size)
+        if !size.include?(:height) || !size.include?(:width)
+          raise ArgumentError, "The `size` dictionary must contain the keys `height` and `width`. Got #{size.keys}"
+        end
+        output_size = [size[:height], size[:width]]
+        ImageTransforms.resize(
+          image,
+          output_size,
+          resample: resample,
+          data_format: data_format,
+          input_data_format: input_data_format,
+          **kwargs
+        )
+      end
+
+      def preprocess(
+        images,
+        do_resize: nil,
+        size: nil,
+        resample: nil,
+        do_rescale: nil,
+        rescale_factor: nil,
+        do_normalize: nil,
+        image_mean: nil,
+        image_std: nil,
+        return_tensors: nil,
+        data_format: ChannelDimension::FIRST,
+        input_data_format: nil,
+        **kwargs
+      )
+        do_resize = !do_resize.nil? ? do_resize : @do_resize
+        do_rescale = !do_rescale.nil? ? do_rescale : @do_rescale
+        do_normalize = !do_normalize.nil? ? do_normalize : @do_normalize
+        resample = !resample.nil? ? resample : @resample
+        rescale_factor = !rescale_factor.nil? ? rescale_factor : @rescale_factor
+        image_mean = !image_mean.nil? ? image_mean : @image_mean
+        image_std = !image_std.nil? ? image_std : @image_std
+
+        size = !size.nil? ? size : @size
+        size_dict = ImageProcessingUtils.get_size_dict(size)
+
+        images = ImageUtils.make_list_of_images(images)
+
+        ImageUtils.validate_kwargs(captured_kwargs: kwargs.keys, valid_processor_keys: @valid_processor_keys)
+
+        if !ImageUtils.valid_images(images)
+          raise ArgumentError,
+            "Invalid image type. Must be of type Vips::Image, Numo::NArray, or Torch::Tensor."
+        end
+        ImageUtils.validate_preprocess_arguments(
+          do_rescale: do_rescale,
+          rescale_factor: rescale_factor,
+          do_normalize: do_normalize,
+          image_mean: image_mean,
+          image_std: image_std,
+          do_resize: do_resize,
+          size: size,
+          resample: resample
+        )
+
+        # All transformations expect numo arrays.
+        images = images.map { |image| ImageUtils.to_numo_array(image) }
+
+        if ImageUtils.is_scaled_image(images[0]) && do_rescale
+          Transformers.logger.warn(
+            "It looks like you are trying to rescale already rescaled images. If the input" +
+            " images have pixel values between 0 and 1, set `do_rescale: false` to avoid rescaling them again."
+          )
+        end
+
+        if input_data_format.nil?
+          # We assume that all images have the same channel dimension format.
+          input_data_format = ImageUtils.infer_channel_dimension_format(images[0])
+        end
+
+        if do_resize
+          images =
+            images.map do |image|
+              resize(image, size_dict, resample: resample, input_data_format: input_data_format)
+            end
+        end
+
+        if do_rescale
+          images =
+            images.map do |image|
+              rescale(image, rescale_factor, input_data_format: input_data_format)
+            end
+        end
+
+        if do_normalize
+          images =
+            images.map do |image|
+              normalize(image, image_mean, image_std, input_data_format: input_data_format)
+            end
+        end
+
+        images =
+          images.map do |image|
+            ImageTransforms.to_channel_dimension_format(image, data_format, input_channel_dim: input_data_format)
+          end
+
+        data = {pixel_values: images}
+        BatchFeature.new(data: data, tensor_type: return_tensors)
+      end
+    end
+  end
+end