transformers-rb 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f29055705824ba101cba238960d4f10825c75bc7867b9eb0b611cda6a547612
-  data.tar.gz: d0967f7742f7b2d6194376eb040a3be81e77a9ded94302aeb934de678959e434
+  metadata.gz: 3f070b9828c5c5ad71c75f46ca9daf1387a5ec3848cb406aac9e5f1bbc1d4531
+  data.tar.gz: 31b28a5a87c58db6fc3146e390e8a4a7bf1ffc34ede6d3cd6fcd7f3aa3df2d28
 SHA512:
-  metadata.gz: 38b9ed4fd654ca593e3d6e7c7f20eb3c6b68ecfa5f86099fbc8d160f9093617cc79a571c331a0ec0c70a6770c8d9460194ba75e61d534f9e15931f22e5ae60c3
-  data.tar.gz: 00ce437ce8fe419fafddd59b7f9f61050d2ddf5817816b53beb1c43badbced9fe77ea28b26f2d607e87a9be855a19c4b254bc50df2881b4126f2def5c6875c3d
+  metadata.gz: aa2055e44b9071a425ebfb59d6b2edbedce1f3cf97e0baa55d1280451c1c1db097a52b0b9615a188b1d96f0854e557fb4cb769b05cb3af4db229cd3fcdf8fb95
+  data.tar.gz: 1af002f238e9189a2e2a6b5f1aafc9201cfd5bc5f8afe4a80b81757b5d9f5d4fa52bc61a57b4fdd6920bd3692f704398aa58c7cc4fd797bd881ab9887c9c77f9

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.1.3 (2024-09-17)
+
+- Added `reranking` pipeline
+- Added DeBERTa-v2
+- Added MPNet
+- Added XLM-RoBERTa
+
 ## 0.1.2 (2024-09-10)
 
 - Fixed default revision for pipelines

data/README.md

@@ -32,11 +32,17 @@ Embedding
 - [intfloat/e5-base-v2](#intfloate5-base-v2)
 - [BAAI/bge-base-en-v1.5](#baaibge-base-en-v15)
 - [Snowflake/snowflake-arctic-embed-m-v1.5](#snowflakesnowflake-arctic-embed-m-v15)
+- [sentence-transformers/all-mpnet-base-v2](#sentence-transformersall-mpnet-base-v2)
 
 Sparse embedding
 
 - [opensearch-project/opensearch-neural-sparse-encoding-v1](#opensearch-projectopensearch-neural-sparse-encoding-v1)
 
+Reranking
+
+- [mixedbread-ai/mxbai-rerank-base-v1](#mixedbread-aimxbai-rerank-base-v1)
+- [BAAI/bge-reranker-base](#baaibge-reranker-base)
+
 ### sentence-transformers/all-MiniLM-L6-v2
 
 [Docs](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
@@ -142,6 +148,17 @@ model = Transformers.pipeline("embedding", "Snowflake/snowflake-arctic-embed-m-v
 embeddings = model.(input, pooling: "cls")
 ```
 
+### sentence-transformers/all-mpnet-base-v2
+
+[Docs](https://huggingface.co/sentence-transformers/all-mpnet-base-v2)
+
+```ruby
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = Transformers.pipeline("embedding", "sentence-transformers/all-mpnet-base-v2")
+embeddings = model.(sentences)
+```
+
 ### opensearch-project/opensearch-neural-sparse-encoding-v1
 
 [Docs](https://huggingface.co/opensearch-project/opensearch-neural-sparse-encoding-v1)
@@ -163,8 +180,37 @@ values[0.., special_token_ids] = 0
 embeddings = values.to_a
 ```
 
+### mixedbread-ai/mxbai-rerank-base-v1
+
+[Docs](https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v1)
+
+```ruby
+query = "How many people live in London?"
+docs = ["Around 9 Million people live in London", "London is known for its financial district"]
+
+model = Transformers.pipeline("reranking", "mixedbread-ai/mxbai-rerank-base-v1")
+result = model.(query, docs)
+```
+
+### BAAI/bge-reranker-base
+
+[Docs](https://huggingface.co/BAAI/bge-reranker-base)
+
+```ruby
+query = "How many people live in London?"
+docs = ["Around 9 Million people live in London", "London is known for its financial district"]
+
+model = Transformers.pipeline("reranking", "BAAI/bge-reranker-base")
+result = model.(query, docs)
+```
+
 ## Pipelines
 
+- [Text](#text)
+- [Vision](#vision)
+
+### Text
+
 Embedding
 
 ```ruby
@@ -172,6 +218,13 @@ embed = Transformers.pipeline("embedding")
 embed.("We are very happy to show you the 🤗 Transformers library.")
 ```
 
+Reranking
+
+```ruby
+rerank = Informers.pipeline("reranking")
+rerank.("Who created Ruby?", ["Matz created Ruby", "Another doc"])
+```
+
 Named-entity recognition
 
 ```ruby
@@ -200,27 +253,32 @@ extractor = Transformers.pipeline("feature-extraction")
 extractor.("We are very happy to show you the 🤗 Transformers library.")
 ```
 
+### Vision
+
 Image classification
 
 ```ruby
 classifier = Transformers.pipeline("image-classification")
-classifier.(URI("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"))
+classifier.("image.jpg")
 ```
 
 Image feature extraction
 
 ```ruby
 extractor = Transformers.pipeline("image-feature-extraction")
-extractor.(URI("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"))
+extractor.("image.jpg")
 ```
 
 ## API
 
-This library follows the [Transformers Python API](https://huggingface.co/docs/transformers/index). Only a few model architectures are currently supported:
+This library follows the [Transformers Python API](https://huggingface.co/docs/transformers/index). The following model architectures are currently supported:
 
 - BERT
+- DeBERTa-v2
 - DistilBERT
+- MPNet
 - ViT
+- XLM-RoBERTa
 
 ## History
 

data/lib/transformers/configuration_utils.rb

@@ -91,10 +91,24 @@ module Transformers
       # Config hash
       @commit_hash = kwargs.delete(:_commit_hash)
 
-      # TODO set kwargs
-      @gradient_checkpointing = kwargs[:gradient_checkpointing]
-      @output_past = kwargs[:output_past]
-      @tie_weights_ = kwargs[:tie_weights_]
+      # Attention implementation to use, if relevant.
+      @attn_implementation_internal = kwargs.delete(:attn_implementation)
+
+      # Drop the transformers version info
+      @transformers_version = kwargs.delete(:transformers_version)
+
+      # Deal with gradient checkpointing
+      # if kwargs[:gradient_checkpointing] == false
+      #   warn(
+      #     "Passing `gradient_checkpointing` to a config initialization is deprecated and will be removed in v5 " +
+      #     "Transformers. Using `model.gradient_checkpointing_enable()` instead, or if you are using the " +
+      #     "`Trainer` API, pass `gradient_checkpointing: true` in your `TrainingArguments`."
+      #   )
+      # end
+
+      kwargs.each do |k, v|
+        instance_variable_set("@#{k}", v)
+      end
     end
 
     def name_or_path
@@ -182,6 +196,20 @@ module Transformers
       JSON.pretty_generate(config_dict.sort_by { |k, _| k }.to_h) + "\n"
     end
 
+    def getattr(key, default)
+      if respond_to?(key)
+        public_send(key)
+      elsif instance_variable_defined?("@#{key}")
+        instance_variable_get("@#{key}")
+      else
+        default
+      end
+    end
+
+    def hasattr(key)
+      respond_to?(key) || instance_variable_defined?("@#{key}")
+    end
+
     class << self
       def from_pretrained(
         pretrained_model_name_or_path,

data/lib/transformers/modeling_utils.rb

@@ -207,7 +207,7 @@ module Transformers
 
     def init_weights
       # Prune heads if needed
-      if @config.pruned_heads
+      if @config.pruned_heads.any?
         prune_heads(@config.pruned_heads)
       end
 
@@ -803,11 +803,18 @@ module Transformers
           raise Todo
         end
 
+        model_class_name = model.class.name.split("::").last
+
         if error_msgs.length > 0
-          raise Todo
+          error_msg = error_msgs.join("\n\t")
+          if error_msg.include?("size mismatch")
+            error_msg += (
+              "\n\tYou may consider adding `ignore_mismatched_sizes: true` in the model `from_pretrained` method."
+            )
+          end
+          raise RuntimeError, "Error(s) in loading state_dict for #{model_class_name}:\n\t#{error_msg}"
         end
 
-        model_class_name = model.class.name.split("::").last
         if unexpected_keys.length > 0
           archs = model.config.architectures.nil? ? [] : model.config.architectures
           warner = archs.include?(model_class_name) ? Transformers.logger.method(:warn) : Transformers.logger.method(:info)

data/lib/transformers/models/auto/auto_factory.rb

@@ -116,7 +116,7 @@ module Transformers
     def _load_attr_from_module(model_type, attr)
       module_name = model_type_to_module_name(model_type)
       if !@modules.include?(module_name)
-        @modules[module_name] = Transformers.const_get(module_name.capitalize)
+        @modules[module_name] = Transformers.const_get(module_name.split("-").map(&:capitalize).join)
       end
       getattribute_from_module(@modules[module_name], attr)
     end

data/lib/transformers/models/auto/configuration_auto.rb

@@ -15,8 +15,11 @@
 module Transformers
   CONFIG_MAPPING_NAMES = {
     "bert" => "BertConfig",
+    "deberta-v2" => "DebertaV2Config",
     "distilbert" => "DistilBertConfig",
-    "vit" => "ViTConfig"
+    "mpnet" => "MPNetConfig",
+    "vit" => "ViTConfig",
+    "xlm-roberta" => "XLMRobertaConfig"
   }
 
   class LazyConfigMapping
@@ -30,7 +33,7 @@ module Transformers
       value = @mapping.fetch(key)
       module_name = model_type_to_module_name(key)
       if !@modules.include?(module_name)
-        @modules[module_name] = Transformers.const_get(module_name.capitalize)
+        @modules[module_name] = Transformers.const_get(module_name.split("-").map(&:capitalize).join)
       end
       @modules[module_name].const_get(value)
     end

data/lib/transformers/models/auto/modeling_auto.rb

@@ -15,16 +15,22 @@
 module Transformers
   MODEL_MAPPING_NAMES = {
     "bert" => "BertModel",
+    "deberta-v2" => "DebertaV2Model",
     "distilbert" => "DistilBertModel",
-    "vit" => "ViTModel"
+    "mpnet" => "MPNetModel",
+    "vit" => "ViTModel",
+    "xlm-roberta" => "XLMRobertaModel"
   }
 
   MODEL_FOR_MASKED_LM_MAPPING_NAMES = {
-    "bert" => "BertForMaskedLM"
+    "bert" => "BertForMaskedLM",
+    "mpnet" => "MPNetForMaskedLM"
   }
 
   MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = {
-    "distilbert" => "DistilBertForSequenceClassification"
+    "deberta-v2" => "DebertaV2ForSequenceClassification",
+    "distilbert" => "DistilBertForSequenceClassification",
+    "xlm-roberta" => "XLMRobertaForSequenceClassification"
   }
 
   MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES = {

data/lib/transformers/models/auto/tokenization_auto.rb

@@ -15,7 +15,10 @@
 module Transformers
   TOKENIZER_MAPPING_NAMES = {
     "bert" => ["BertTokenizer", "BertTokenizerFast"],
-    "distilbert" => ["DistilBertTokenizer", "DistilBertTokenizerFast"]
+    "deberta-v2" => ["DebertaV2TokenizerFast"],
+    "distilbert" => ["DistilBertTokenizer", "DistilBertTokenizerFast"],
+    "mpnet" => ["MPNetTokenizerFast"],
+    "xlm-roberta" => ["XLMRobertaTokenizerFast"]
   }
 
   TOKENIZER_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, TOKENIZER_MAPPING_NAMES)
@@ -98,7 +101,7 @@ module Transformers
 
         TOKENIZER_MAPPING_NAMES.each do |module_name, tokenizers|
           if tokenizers.include?(class_name)
-            cls = Transformers.const_get(module_name.capitalize).const_get(class_name)
+            cls = Transformers.const_get(module_name.split("-").map(&:capitalize).join).const_get(class_name)
             raise Error, "Invalid tokenizer class: #{class_name}" unless cls < PreTrainedTokenizer || cls < PreTrainedTokenizerFast
             return cls
           end

data/lib/transformers/models/deberta_v2/configuration_deberta_v2.rb

@@ -0,0 +1,80 @@
+# Copyright 2020, Microsoft and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module DebertaV2
+    class DebertaV2Config < PretrainedConfig
+      self.model_type = "deberta-v2"
+
+      attr_reader :vocab_size, :hidden_size, :num_hidden_layers, :num_attention_heads,
+        :intermediate_size, :hidden_act, :hidden_dropout_prob, :attention_probs_dropout_prob,
+        :max_position_embeddings, :type_vocab_size, :initializer_range, :layer_norm_eps,
+        :relative_attention, :max_relative_positions, :pad_token_id, :position_biased_input,
+        :pos_att_type, :pooler_dropout, :pooler_hidden_act, :pooler_hidden_size
+
+      def initialize(
+        vocab_size: 128100,
+        hidden_size: 1536,
+        num_hidden_layers: 24,
+        num_attention_heads: 24,
+        intermediate_size: 6144,
+        hidden_act: "gelu",
+        hidden_dropout_prob: 0.1,
+        attention_probs_dropout_prob: 0.1,
+        max_position_embeddings: 512,
+        type_vocab_size: 0,
+        initializer_range: 0.02,
+        layer_norm_eps: 1e-07,
+        relative_attention: false,
+        max_relative_positions: -1,
+        pad_token_id: 0,
+        position_biased_input: true,
+        pos_att_type: nil,
+        pooler_dropout: 0,
+        pooler_hidden_act: "gelu",
+        **kwargs
+      )
+        super(**kwargs)
+
+        @hidden_size = hidden_size
+        @num_hidden_layers = num_hidden_layers
+        @num_attention_heads = num_attention_heads
+        @intermediate_size = intermediate_size
+        @hidden_act = hidden_act
+        @hidden_dropout_prob = hidden_dropout_prob
+        @attention_probs_dropout_prob = attention_probs_dropout_prob
+        @max_position_embeddings = max_position_embeddings
+        @type_vocab_size = type_vocab_size
+        @initializer_range = initializer_range
+        @relative_attention = relative_attention
+        @max_relative_positions = max_relative_positions
+        @pad_token_id = pad_token_id
+        @position_biased_input = position_biased_input
+
+        # Backwards compatibility
+        if pos_att_type.is_a?(String)
+          pos_att_type = pos_att_type.downcase.split("|").map { |x| x.strip }
+        end
+
+        @pos_att_type = pos_att_type
+        @vocab_size = vocab_size
+        @layer_norm_eps = layer_norm_eps
+
+        @pooler_hidden_size = kwargs[:pooler_hidden_size] || hidden_size
+        @pooler_dropout = pooler_dropout
+        @pooler_hidden_act = pooler_hidden_act
+      end
+    end
+  end
+end