transformers-rb 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f070b9828c5c5ad71c75f46ca9daf1387a5ec3848cb406aac9e5f1bbc1d4531
-  data.tar.gz: 31b28a5a87c58db6fc3146e390e8a4a7bf1ffc34ede6d3cd6fcd7f3aa3df2d28
+  metadata.gz: 16b409e954f6bcc45fd4f3f2db94dc92c87d47c4c06936162978cc7d7e54fc09
+  data.tar.gz: c40ea58b7531e89a041ce2782dea89536f85227903e2dc9a60113afe041bb9f7
 SHA512:
-  metadata.gz: aa2055e44b9071a425ebfb59d6b2edbedce1f3cf97e0baa55d1280451c1c1db097a52b0b9615a188b1d96f0854e557fb4cb769b05cb3af4db229cd3fcdf8fb95
-  data.tar.gz: 1af002f238e9189a2e2a6b5f1aafc9201cfd5bc5f8afe4a80b81757b5d9f5d4fa52bc61a57b4fdd6920bd3692f704398aa58c7cc4fd797bd881ab9887c9c77f9
+  metadata.gz: 0576500ca9fe9379aae4c2cc050aa34c90eea7b2d5251b6139c48d88b5107086e6197aad5210bc74d7969a5bfa0458d31ce7faf561df69b2ae9b2a6400280ce0
+  data.tar.gz: b5b0a865acdd37bcde11571e365a1d39b473f3dcbbe737a082f37002f886fe38f0f57c77f44236f859939d1d3b1df9659bb524a483cd0b9048dbc4cd472a3355

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.4 (2024-10-22)
+
+- Added `BertForSequenceClassification`
+
 ## 0.1.3 (2024-09-17)
 
 - Added `reranking` pipeline

data/README.md

@@ -27,12 +27,13 @@ Embedding
 
 - [sentence-transformers/all-MiniLM-L6-v2](#sentence-transformersall-MiniLM-L6-v2)
 - [sentence-transformers/multi-qa-MiniLM-L6-cos-v1](#sentence-transformersmulti-qa-MiniLM-L6-cos-v1)
+- [sentence-transformers/all-mpnet-base-v2](#sentence-transformersall-mpnet-base-v2)
+- [sentence-transformers/paraphrase-MiniLM-L6-v2](#sentence-transformersparaphrase-minilm-l6-v2)
 - [mixedbread-ai/mxbai-embed-large-v1](#mixedbread-aimxbai-embed-large-v1)
 - [thenlper/gte-small](#thenlpergte-small)
 - [intfloat/e5-base-v2](#intfloate5-base-v2)
 - [BAAI/bge-base-en-v1.5](#baaibge-base-en-v15)
 - [Snowflake/snowflake-arctic-embed-m-v1.5](#snowflakesnowflake-arctic-embed-m-v15)
-- [sentence-transformers/all-mpnet-base-v2](#sentence-transformersall-mpnet-base-v2)
 
 Sparse embedding
 
@@ -69,6 +70,28 @@ scores = doc_embeddings.map { |e| e.zip(query_embedding).sum { |d, q| d * q } }
 doc_score_pairs = docs.zip(scores).sort_by { |d, s| -s }
 ```
 
+### sentence-transformers/all-mpnet-base-v2
+
+[Docs](https://huggingface.co/sentence-transformers/all-mpnet-base-v2)
+
+```ruby
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = Transformers.pipeline("embedding", "sentence-transformers/all-mpnet-base-v2")
+embeddings = model.(sentences)
+```
+
+### sentence-transformers/paraphrase-MiniLM-L6-v2
+
+[Docs](https://huggingface.co/sentence-transformers/paraphrase-MiniLM-L6-v2)
+
+```ruby
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = Transformers.pipeline("embedding", "sentence-transformers/paraphrase-MiniLM-L6-v2")
+embeddings = model.(sentences)
+```
+
 ### mixedbread-ai/mxbai-embed-large-v1
 
 [Docs](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1)
@@ -148,17 +171,6 @@ model = Transformers.pipeline("embedding", "Snowflake/snowflake-arctic-embed-m-v
 embeddings = model.(input, pooling: "cls")
 ```
 
-### sentence-transformers/all-mpnet-base-v2
-
-[Docs](https://huggingface.co/sentence-transformers/all-mpnet-base-v2)
-
-```ruby
-sentences = ["This is an example sentence", "Each sentence is converted"]
-
-model = Transformers.pipeline("embedding", "sentence-transformers/all-mpnet-base-v2")
-embeddings = model.(sentences)
-```
-
 ### opensearch-project/opensearch-neural-sparse-encoding-v1
 
 [Docs](https://huggingface.co/opensearch-project/opensearch-neural-sparse-encoding-v1)

data/lib/transformers/configuration_utils.rb

@@ -36,7 +36,9 @@ module Transformers
 
     attr_reader :output_hidden_states, :output_attentions, :pruned_heads, :tie_word_embeddings, :tokenizer_class,
       :chunk_size_feed_forward, :pad_token_id, :is_decoder, :add_cross_attention,
-      :problem_type, :id2label, :architectures, :is_encoder_decoder, :tie_encoder_decoder, :_commit_hash
+      :id2label, :architectures, :is_encoder_decoder, :tie_encoder_decoder, :_commit_hash
+
+    attr_accessor :problem_type
 
     def initialize(**kwargs)
       @return_dict = kwargs.delete(:return_dict) { true }

data/lib/transformers/hf_hub/constants.rb

@@ -45,7 +45,7 @@ module Transformers
 
     REPO_TYPES_URL_PREFIXES = {
       REPO_TYPE_DATASET => "datasets/",
-      REPO_TYPE_SPACE => "spaces/",
+      REPO_TYPE_SPACE => "spaces/"
     }
 
     # default cache

data/lib/transformers/modeling_utils.rb

@@ -23,7 +23,7 @@ module Transformers
       "xavier_uniform!" => Torch::NN::Init.method(:xavier_uniform!),
       "xavier_normal!" => Torch::NN::Init.method(:xavier_normal!),
       "kaiming_uniform!" => Torch::NN::Init.method(:kaiming_uniform!),
-      "kaiming_normal!" => Torch::NN::Init.method(:kaiming_normal!),
+      "kaiming_normal!" => Torch::NN::Init.method(:kaiming_normal!)
       # "uniform" => Torch::NN::Init.method(:uniform),
       # "normal" => Torch::NN::Init.method(:normal),
       # "xavier_uniform" => Torch::NN::Init.method(:xavier_uniform),

data/lib/transformers/models/auto/configuration_auto.rb

@@ -55,7 +55,7 @@ module Transformers
       config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
       if config_dict[:model_type]
         config_class = CONFIG_MAPPING[config_dict[:model_type]]
-        return config_class.from_dict(config_dict, **unused_kwargs)
+        config_class.from_dict(config_dict, **unused_kwargs)
       else
         raise Todo
       end

data/lib/transformers/models/auto/modeling_auto.rb

@@ -28,6 +28,7 @@ module Transformers
   }
 
   MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = {
+    "bert" => "BertForSequenceClassification",
     "deberta-v2" => "DebertaV2ForSequenceClassification",
     "distilbert" => "DistilBertForSequenceClassification",
     "xlm-roberta" => "XLMRobertaForSequenceClassification"

data/lib/transformers/models/bert/modeling_bert.rb

@@ -353,7 +353,7 @@ module Transformers
       def feed_forward_chunk(attention_output)
         intermediate_output = @intermediate.(attention_output)
         layer_output = @output.(intermediate_output, attention_output)
-        return layer_output
+        layer_output
       end
     end
 
@@ -370,7 +370,7 @@ module Transformers
         attention_mask: nil,
         head_mask: nil,
         encoder_hidden_states: nil,
-        encoder_attention_mask:nil,
+        encoder_attention_mask: nil,
         past_key_values: nil,
         use_cache: nil,
         output_attentions: false,
@@ -814,7 +814,7 @@ module Transformers
         loss = nil
         if !labels.nil?
           loss_fct = CrossEntropyLoss.new
-          loss = loss_fct.(logits.view(-1,@num_labels), labels.view(-1))
+          loss = loss_fct.(logits.view(-1, @num_labels), labels.view(-1))
         end
 
         if !return_dict
@@ -829,8 +829,98 @@ module Transformers
         )
       end
     end
+
+    class BertForSequenceClassification < BertPreTrainedModel
+      def initialize(config)
+        super
+        @num_labels = config.num_labels
+        @config = config
+
+        @bert = BertModel.new(config, add_pooling_layer: true)
+        classifier_dropout = (
+          config.classifier_dropout.nil? ? config.hidden_dropout_prob : config.classifier_dropout
+        )
+        @dropout = Torch::NN::Dropout.new(p: classifier_dropout)
+        @classifier = Torch::NN::Linear.new(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        post_init
+      end
+
+      def forward(
+        input_ids: nil,
+        attention_mask: nil,
+        token_type_ids: nil,
+        position_ids: nil,
+        head_mask: nil,
+        inputs_embeds: nil,
+        labels: nil,
+        output_attentions: nil,
+        output_hidden_states: nil,
+        return_dict: nil
+      )
+        return_dict = @config.use_return_dict if return_dict.nil?
+
+        outputs = @bert.(
+          input_ids: input_ids,
+          attention_mask: attention_mask,
+          token_type_ids: token_type_ids,
+          position_ids: position_ids,
+          head_mask: head_mask,
+          inputs_embeds: inputs_embeds,
+          output_attentions: output_attentions,
+          output_hidden_states: output_hidden_states,
+          return_dict: return_dict
+        )
+
+        pooled_output = outputs[1]
+
+        pooled_output = @dropout.(pooled_output)
+        logits = @classifier.(pooled_output)
+
+        loss = nil
+        if !labels.nil?
+          if @config.problem_type.nil?
+            if @num_labels == 1
+              @config.problem_type = "regression"
+            elsif @num_labels > 1 && (labels.dtype == Torch.long || labels.dtype == Torch.int)
+              @config.problem_type = "single_label_classification"
+            else
+              @config.problem_type = "multi_label_classification"
+            end
+          end
+
+          if @config.problem_type == "regression"
+            loss_fct = Torch::NN::MSELoss.new
+            if @num_labels == 1
+              loss = loss_fct.(logits.squeeze, labels.squeeze)
+            else
+              loss = loss_fct.(logits, labels)
+            end
+          elsif @config.problem_type == "single_label_classification"
+            loss_fct = Torch::NN::CrossEntropyLoss.new
+            loss = loss_fct.(logits.view(-1, @num_labels), labels.view(-1))
+          elsif @config.problem_type == "multi_label_classification"
+            loss_fct = Torch::NN::BCEWithLogitsLoss.new
+            loss = loss_fct.(logits, labels)
+          end
+        end
+
+        if !return_dict
+          raise Todo
+        end
+
+        SequenceClassifierOutput.new(
+          loss: loss,
+          logits: logits,
+          hidden_states: outputs.hidden_states,
+          attentions: outputs.attentions
+        )
+      end
+    end
   end
 
   BertModel = Bert::BertModel
   BertForTokenClassification = Bert::BertForTokenClassification
+  BertForSequenceClassification = Bert::BertForSequenceClassification
 end

data/lib/transformers/models/distilbert/configuration_distilbert.rb

@@ -19,7 +19,7 @@ module Transformers
       self.attribute_map = {
         hidden_size: "dim",
         num_attention_heads: "n_heads",
-        num_hidden_layers: "n_layers",
+        num_hidden_layers: "n_layers"
       }
 
      attr_reader :vocab_size, :max_position_embeddings, :sinusoidal_pos_embds, :n_layers, :n_heads,

data/lib/transformers/tokenization_utils.rb

@@ -15,7 +15,6 @@
 module Transformers
   class PreTrainedTokenizer < PreTrainedTokenizerBase
     def initialize(**kwargs)
-
       # 2. init `_added_tokens_decoder` if child class did not
       if !instance_variable_defined?(:@added_tokens_decoder)
         @added_tokens_decoder = {}

data/lib/transformers/utils/hub.rb

@@ -181,9 +181,9 @@ module Transformers
               proxies: proxies,
               timeout: 10
             )
-            return true
+            true
           rescue HfHub::OfflineModeIsEnabled
-            return has_file_in_cache
+            has_file_in_cache
           rescue HfHub::GatedRepoError => e
             Transformers.logger.error(e)
             raise EnvironmentError,
@@ -200,7 +200,7 @@ module Transformers
               "#{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for this " +
               "model name. Check the model page at 'https://huggingface.co/#{path_or_repo}' for available revisions."
           rescue HfHub::EntryNotFoundError
-            return false  # File does not exist
+            false  # File does not exist
           end
         end
       end

data/lib/transformers/version.rb

@@ -1,3 +1,3 @@
 module Transformers
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: transformers-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-17 00:00:00.000000000 Z
+date: 2024-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray