transformers-rb 0.1.0

data/lib/transformers/configuration_utils.rb

@@ -0,0 +1,285 @@
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  class PretrainedConfig
+    extend ClassAttribute
+
+    class_attribute :model_type, ""
+    class_attribute :attribute_map, {}
+
+    # TODO support setter
+    def method_missing(m, *args, **kwargs)
+      if self.class.attribute_map.include?(m)
+        instance_variable_get("@#{self.class.attribute_map[m]}")
+      else
+        super
+      end
+    end
+
+    # TODO support setter
+    def respond_to_missing?(m, include_private = true)
+      self.class.attribute_map.include?(m) || super
+    end
+
+    attr_reader :output_hidden_states, :output_attentions, :pruned_heads, :tie_word_embeddings, :tokenizer_class,
+      :chunk_size_feed_forward, :pad_token_id, :is_decoder, :add_cross_attention,
+      :problem_type, :id2label, :architectures, :is_encoder_decoder, :tie_encoder_decoder, :_commit_hash
+
+    def initialize(**kwargs)
+      @return_dict = kwargs.delete(:return_dict) { true }
+      @output_hidden_states = kwargs.delete(:output_hidden_states) { false }
+      @output_attentions = kwargs.delete(:output_attentions) { false }
+      @pruned_heads = kwargs.delete(:pruned_heads) { {} }
+      @tie_word_embeddings = kwargs.delete(:tie_word_embeddings) { true }
+      @chunk_size_feed_forward = kwargs.delete(:chunk_size_feed_forward) { 0 }
+
+      # Is decoder is used in encoder-decoder models to differentiate encoder from decoder
+      @is_encoder_decoder = kwargs.delete(:is_encoder_decoder) { false }
+      @is_decoder = kwargs.delete(:is_decoder) { false }
+      @cross_attention_hidden_size = kwargs.delete(:cross_attention_hidden_size)
+      @add_cross_attention = kwargs.delete(:add_cross_attention) { false }
+      @tie_encoder_decoder = kwargs.delete(:tie_encoder_decoder) { false }
+
+      # Fine-tuning task arguments
+      @architectures = kwargs.delete(:architectures)
+      @finetuning_task = kwargs.delete(:finetuning_task)
+      @id2label = kwargs.delete(:id2label)
+      @label2id = kwargs.delete(:label2id)
+      if !@label2id.nil? && !@label2id.is_a?(Hash)
+        raise ArgumentError, "Argument label2id should be a dictionary."
+      end
+      if !@id2label.nil?
+        if !@id2label.is_a?(Hash)
+          raise ArgumentError, "Argument id2label should be a dictionary."
+        end
+        num_labels = kwargs.delete(:num_labels)
+        if !num_labels.nil? && id2label.length != num_labels
+          raise Todo
+        end
+        @id2label = @id2label.transform_keys(&:to_i)
+        # Keys are always strings in JSON so convert ids to int here.
+      else
+        self.num_labels = kwargs.delete(:num_labels) { 2 }
+      end
+
+      # Tokenizer arguments TODO: eventually tokenizer and models should share the same config
+      @tokenizer_class = kwargs.delete(:tokenizer_class)
+      @prefix = kwargs.delete(:prefix)
+      @bos_token_id = kwargs.delete(:bos_token_id)
+      @pad_token_id = kwargs.delete(:pad_token_id)
+      @eos_token_id = kwargs.delete(:eos_token_id)
+      @sep_token_id = kwargs.delete(:sep_token_id)
+
+      # regression / multi-label classification
+      @problem_type = kwargs.delete(:problem_type)
+
+      # Name or path to the pretrained checkpoint
+      @name_or_path = kwargs.delete(:name_or_path).to_s
+      # Config hash
+      @commit_hash = kwargs.delete(:_commit_hash)
+
+      # TODO set kwargs
+      @gradient_checkpointing = kwargs[:gradient_checkpointing]
+      @output_past = kwargs[:output_past]
+      @tie_weights_ = kwargs[:tie_weights_]
+    end
+
+    def name_or_path
+      @name_or_path
+    end
+
+    def name_or_path=(value)
+      @name_or_path = value.to_s
+    end
+
+    def num_labels
+      @id2label.length
+    end
+
+    def num_labels=(num_labels)
+      if @id2label.nil? || @id2label.length != num_labels
+        @id2label = num_labels.times.to_h { |i| [i, "LABEL_#{i}"] }
+        @label2id =  @id2label.invert
+      end
+    end
+
+    def _attn_implementation
+      # This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.)
+      if instance_variable_defined?(:@attn_implementation_internal)
+        if instance_variable_get(:@attn_implementation_internal).nil?
+          # `config.attn_implementation` should never be None, for backward compatibility.
+          "eager"
+        else
+          @attn_implementation_internal
+        end
+      else
+        "eager"
+      end
+    end
+
+    def use_return_dict
+      @return_dict
+    end
+
+    def to_s
+      "#{self.class.name} #{to_json_string}"
+    end
+
+    def to_diff_dict
+      config_dict = to_dict
+
+      # get the default config dict
+      default_config_dict = PretrainedConfig.new.to_dict
+
+      serializable_config_dict = {}
+
+      config_dict.each do |key, value|
+        key = :_name_or_path if key == :name_or_path
+        if !default_config_dict.include?(key) || value != default_config_dict[key] || key == :transformers_version
+          serializable_config_dict[key] = value
+        end
+      end
+
+      serializable_config_dict
+    end
+
+    def _dict
+      instance_variables.to_h { |k| [k[1..].to_sym, instance_variable_get(k)] }
+    end
+
+    def to_dict
+      output = Copy.deepcopy(_dict)
+      output[:model_type] = self.class.model_type
+      output.delete(:_auto_class)
+      output.delete(:_commit_hash)
+      output.delete(:_attn_implementation_internal)
+
+      # Transformers version when serializing the model
+      output[:transformers_version] = VERSION
+
+      output
+    end
+
+    def to_json_string(use_diff: true)
+      if use_diff == true
+        config_dict = to_diff_dict
+      else
+        config_dict = to_dict
+      end
+      JSON.pretty_generate(config_dict.sort_by { |k, _| k }.to_h) + "\n"
+    end
+
+    class << self
+      def from_pretrained(
+        pretrained_model_name_or_path,
+        cache_dir: nil,
+        force_download: false,
+        local_files_only: false,
+        token: nil,
+        revision: "main",
+        **kwargs
+      )
+        config_dict, kwargs = get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        from_dict(config_dict, **kwargs)
+      end
+
+      def from_dict(config_dict, **kwargs)
+        return_unused_kwargs = kwargs.delete(:return_unused_kwargs) { false }
+
+        # The commit hash might have been updated in the `config_dict`, we don't want the kwargs to erase that update.
+        if kwargs.include?(:_commit_hash) && config_dict.include?(:_commit_hash)
+          kwargs[:_commit_hash] = config_dict[:_commit_hash]
+        end
+
+        config = new(**config_dict)
+
+        kwargs.each do |key, value|
+          if config.respond_to?("#{key}=")
+            config.public_send("#{key}=", value)
+          end
+        end
+
+        Transformers.logger.info("Model config #{config}")
+        if return_unused_kwargs
+          [config, kwargs]
+        else
+          config
+        end
+      end
+
+      def get_config_dict(pretrained_model_name_or_path, **kwargs)
+        # Get config dict associated with the base config file
+        config_dict, kwargs = _get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        [config_dict, kwargs]
+      end
+
+      private
+
+      def _get_config_dict(pretrained_model_name_or_path, **kwargs)
+        cache_dir = kwargs.delete(:cache_dir)
+        force_download = kwargs.delete(:force_download) { false }
+        resume_download = kwargs.delete(:resume_download) { false }
+        proxies = kwargs.delete(:proxies)
+        token = kwargs.delete(:token)
+        local_files_only = kwargs.delete(:local_files_only) { false }
+        revision = kwargs.delete(:revision)
+        _trust_remote_code = kwargs.delete(:trust_remote_code)
+        subfolder = kwargs.delete(:subfolder) { "" }
+        _from_pipeline = kwargs.delete(:_from_pipeline)
+        from_auto_class = kwargs.delete(:_from_auto) { false }
+        commit_hash = kwargs.delete(:_commit_hash)
+
+        user_agent = {file_type: "config", from_auto_class: from_auto_class}
+
+        is_local = Dir.exist?(pretrained_model_name_or_path)
+        configuration_file = kwargs.delete(:_configuration_file) || CONFIG_NAME
+
+        resolved_config_file = Utils::Hub.cached_file(
+          pretrained_model_name_or_path,
+          configuration_file,
+          cache_dir: cache_dir,
+          force_download: force_download,
+          proxies: proxies,
+          resume_download: resume_download,
+          local_files_only: local_files_only,
+          token: token,
+          user_agent: user_agent,
+          revision: revision,
+          subfolder: subfolder,
+          _commit_hash: commit_hash
+        )
+        commit_hash = Utils::Hub.extract_commit_hash(resolved_config_file, commit_hash)
+
+        config_dict = _dict_from_json_file(resolved_config_file)
+        config_dict[:_commit_hash] = commit_hash
+
+        if is_local
+          Transformers.logger.info("loading configuration file #{resolved_config_file}")
+        else
+          Transformers.logger.info("loading configuration file #{configuration_file} from cache at #{resolved_config_file}")
+        end
+
+        [config_dict, kwargs]
+      end
+
+      def _dict_from_json_file(json_file)
+        JSON.load_file(json_file).transform_keys(&:to_sym)
+      end
+    end
+  end
+end

data/lib/transformers/convert_slow_tokenizer.rb

@@ -0,0 +1,90 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module ConvertSlowTokenizer
+    class Converter
+      def initialize(original_tokenizer)
+        @original_tokenizer = original_tokenizer
+      end
+
+      def converted
+        raise NotImplementedError
+      end
+    end
+
+    class BertConverter < Converter
+      def converted
+        vocab = @original_tokenizer.vocab
+        tokenizer = Tokenizers::Tokenizer.new(Tokenizers::Models::WordPiece.new(vocab: vocab, unk_token: @original_tokenizer.unk_token.to_s))
+
+        tokenize_chinese_chars = false
+        strip_accents = false
+        do_lower_case = false
+        if @original_tokenizer.basic_tokenizer
+          tokenize_chinese_chars = @original_tokenizer.basic_tokenizer.tokenize_chinese_chars
+          strip_accents = @original_tokenizer.basic_tokenizer.strip_accents
+          do_lower_case = @original_tokenizer.basic_tokenizer.do_lower_case
+        end
+
+        tokenizer.normalizer =
+          Tokenizers::Normalizers::BertNormalizer.new(
+            clean_text: true,
+            handle_chinese_chars: tokenize_chinese_chars,
+            strip_accents: strip_accents,
+            lowercase: do_lower_case,
+          )
+        tokenizer.pre_tokenizer = Tokenizers::PreTokenizers::BertPreTokenizer.new
+
+        cls = @original_tokenizer.cls_token.to_s
+        sep = @original_tokenizer.sep_token.to_s
+        cls_token_id = @original_tokenizer.cls_token_id
+        sep_token_id = @original_tokenizer.sep_token_id
+
+        tokenizer.post_processor =
+          Tokenizers::Processors::TemplateProcessing.new(
+            single: "#{cls}:0 $A:0 #{sep}:0",
+            pair: "#{cls}:0 $A:0 #{sep}:0 $B:1 #{sep}:1",
+            special_tokens: [
+              [cls, cls_token_id],
+              [sep, sep_token_id]
+            ]
+          )
+        tokenizer.decoder = Tokenizers::Decoders::WordPiece.new(prefix: "##")
+
+        tokenizer
+      end
+    end
+
+    SLOW_TO_FAST_CONVERTERS = {
+      "BertTokenizer" => BertConverter,
+      "DistilBertTokenizer" => BertConverter
+    }
+
+    def self.convert_slow_tokenizer(transformer_tokenizer)
+      tokenizer_class_name = transformer_tokenizer.class.name.split("::").last
+
+      if !SLOW_TO_FAST_CONVERTERS.include?(tokenizer_class_name)
+        raise ArgumentError,
+          "An instance of tokenizer class #{tokenizer_class_name} cannot be converted in a Fast tokenizer instance." +
+          " No converter was found. Currently available slow->fast convertors:" +
+          " #{SLOW_TO_FAST_CONVERTERS.keys}"
+      end
+
+      converter_class = SLOW_TO_FAST_CONVERTERS.fetch(tokenizer_class_name)
+
+      converter_class.new(transformer_tokenizer).converted
+    end
+  end
+end

data/lib/transformers/data/processors/squad.rb

@@ -0,0 +1,115 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  class SquadExample
+    attr_reader :question_text, :context_text
+
+    def initialize(
+      qas_id,
+      question_text,
+      context_text,
+      answer_text,
+      start_position_character,
+      title,
+      answers: [],
+      is_impossible: false
+    )
+      @qas_id = qas_id
+      @question_text = question_text
+      @context_text = context_text
+      @answer_text = answer_text
+      @title = title
+      @is_impossible = is_impossible
+      @answers = answers
+
+      @start_position, @end_position = 0, 0
+
+      doc_tokens = []
+      char_to_word_offset = []
+      prev_is_whitespace = true
+
+      # Split on whitespace so that different tokens may be attributed to their original position.
+      @context_text.each_char do |c|
+        if _is_whitespace(c)
+          prev_is_whitespace = true
+        else
+          if prev_is_whitespace
+            doc_tokens << c
+          else
+            doc_tokens[-1] += c
+          end
+          prev_is_whitespace = false
+        end
+        char_to_word_offset << (doc_tokens.length - 1)
+      end
+
+      @doc_tokens = doc_tokens
+      @char_to_word_offset = char_to_word_offset
+
+      # Start and end positions only has a value during evaluation.
+      if !start_position_character.nil? && !is_impossible
+        @start_position = char_to_word_offset[start_position_character]
+        @end_position = char_to_word_offset[
+          [start_position_character + answer_text.length - 1, char_to_word_offset.length - 1].min
+        ]
+      end
+    end
+
+    def _is_whitespace(c)
+      c == " " || c == "\t" || c == "\r" || c == "\n" || c.ord == 0x202F
+    end
+  end
+
+  class SquadFeatures
+    def initialize(
+      input_ids:,
+      attention_mask:,
+      token_type_ids:,
+      cls_index:,
+      p_mask:,
+      example_index:,
+      unique_id:,
+      paragraph_len:,
+      token_is_max_context:,
+      tokens:,
+      token_to_orig_map:,
+      start_position:,
+      end_position:,
+      is_impossible:,
+      qas_id: nil,
+      encoding: nil
+    )
+      @input_ids = input_ids
+      @attention_mask = attention_mask
+      @token_type_ids = token_type_ids
+      @cls_index = cls_index
+      @p_mask = p_mask
+
+      @example_index = example_index
+      @unique_id = unique_id
+      @paragraph_len = paragraph_len
+      @token_is_max_context = token_is_max_context
+      @tokens = tokens
+      @token_to_orig_map = token_to_orig_map
+
+      @start_position = start_position
+      @end_position = end_position
+      @is_impossible = is_impossible
+      @qas_id = qas_id
+
+      @encoding = encoding
+    end
+  end
+end

data/lib/transformers/dynamic_module_utils.rb

@@ -0,0 +1,25 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module DynamicModuleUtils
+    # TODO improve
+    def self.resolve_trust_remote_code(trust_remote_code, model_name, has_local_code, has_remote_code)
+      if trust_remote_code
+        raise Error, "trust_remote_code not supported"
+      end
+      trust_remote_code
+    end
+  end
+end

data/lib/transformers/feature_extraction_utils.rb

@@ -0,0 +1,110 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  class BatchFeature
+    def initialize(data:, tensor_type:)
+      @data = data
+      convert_to_tensors(tensor_type: tensor_type)
+    end
+
+    def to_h
+      @data
+    end
+
+    def [](item)
+      @data[item]
+    end
+
+    def keys
+      @data.keys
+    end
+
+    def values
+      @data.values
+    end
+
+    def items
+      @data
+    end
+
+    def _get_is_as_tensor_fns(tensor_type: nil)
+      if tensor_type.nil?
+        return [nil, nil]
+      end
+
+      as_tensor = lambda do |value|
+        if value.is_a?(Array) && value.length > 0 && value[0].is_a?(Numo::NArray)
+          value = Numo::NArray.cast(value)
+        end
+        Torch.tensor(value)
+      end
+
+      is_tensor = Torch.method(:tensor?)
+
+      [is_tensor, as_tensor]
+    end
+
+    def convert_to_tensors(tensor_type: nil)
+      if tensor_type.nil?
+        return self
+      end
+
+      is_tensor, as_tensor = _get_is_as_tensor_fns(tensor_type: tensor_type)
+
+      # Do the tensor conversion in batch
+      items.each do |key, value|
+        begin
+          if !is_tensor.(value)
+            tensor = as_tensor.(value)
+
+            @data[key] = tensor
+          end
+        rescue
+          if key == :overflowing_values
+            raise ArgumentError, "Unable to create tensor returning overflowing values of different lengths."
+          end
+          raise ArgumentError,
+            "Unable to create tensor, you should probably activate padding " +
+            "with 'padding: true' to have batched tensors with the same length."
+        end
+      end
+
+      self
+    end
+
+    def to(*args, **kwargs)
+      new_data = {}
+      device = kwargs[:device]
+      # Check if the args are a device or a dtype
+      if device.nil? && args.length > 0
+        raise Todo
+      end
+      # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
+      items.each do |k, v|
+        # check if v is a floating point
+        if Torch.floating_point?(v)
+          # cast and send to device
+          new_data[k] = v.to(*args, **kwargs)
+        elsif !device.nil?
+          new_data[k] = v.to(device)
+        else
+          new_data[k] = v
+        end
+      end
+      @data = new_data
+      self
+    end
+  end
+end

data/lib/transformers/hf_hub/constants.rb

@@ -0,0 +1,71 @@
+module Transformers
+  module HfHub
+    # Possible values for env variables
+
+    ENV_VARS_TRUE_VALUES = ["1", "ON", "YES", "TRUE"]
+    ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES + ["AUTO"]
+
+    def self._is_true(value)
+      if value.nil?
+        return false
+      end
+      ENV_VARS_TRUE_VALUES.include?(value.upcase)
+    end
+
+    def self._as_int(value)
+      if value.nil?
+        return nil
+      end
+      value.to_i
+    end
+
+    # Constants for file downloads
+
+    DEFAULT_ETAG_TIMEOUT = 10
+
+    # Git-related constants
+
+    DEFAULT_REVISION = "main"
+
+    ENDPOINT = ENV["HF_ENDPOINT"] || "https://huggingface.co"
+
+    HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/%{repo_id}/resolve/%{revision}/%{filename}"
+    HUGGINGFACE_HEADER_X_REPO_COMMIT = "x-repo-commit"
+    HUGGINGFACE_HEADER_X_LINKED_ETAG = "x-linked-etag"
+    HUGGINGFACE_HEADER_X_LINKED_SIZE = "x-linked-size"
+
+    REPO_ID_SEPARATOR = "--"
+    # ^ this substring is not allowed in repo_ids on hf.co
+    # and is the canonical one we use for serialization of repo ids elsewhere.
+
+    REPO_TYPE_DATASET = "dataset"
+    REPO_TYPE_SPACE = "space"
+    REPO_TYPE_MODEL = "model"
+    REPO_TYPES = [nil, REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_SPACE]
+
+    REPO_TYPES_URL_PREFIXES = {
+      REPO_TYPE_DATASET => "datasets/",
+      REPO_TYPE_SPACE => "spaces/",
+    }
+
+    # default cache
+    DEFAULT_HOME = File.join(ENV.fetch("HOME"), ".cache")
+    HF_HOME =
+      File.expand_path(
+        ENV.fetch(
+          "HF_HOME",
+          File.join(ENV.fetch("XDG_CACHE_HOME", DEFAULT_HOME), "huggingface")
+        )
+      )
+
+    # New env variables
+    HF_HUB_CACHE = ENV["HF_HUB_CACHE"] || File.join(HF_HOME, "hub")
+
+    HF_HUB_OFFLINE = _is_true(ENV["HF_HUB_OFFLINE"] || ENV["TRANSFORMERS_OFFLINE"])
+
+    # Disable sending the cached token by default is all HTTP requests to the Hub
+    HF_HUB_DISABLE_IMPLICIT_TOKEN = _is_true(ENV["HF_HUB_DISABLE_IMPLICIT_TOKEN"])
+
+    HF_HUB_ENABLE_HF_TRANSFER = _is_true(ENV["HF_HUB_ENABLE_HF_TRANSFER"])
+  end
+end