transformers-rb 0.1.2 → 0.1.3

data/lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb

@@ -0,0 +1,68 @@
+# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+module Transformers
+  module XlmRoberta
+    class XLMRobertaTokenizerFast < PreTrainedTokenizerFast
+      VOCAB_FILES_NAMES = {vocab_file: "sentencepiece.bpe.model", tokenizer_file: "tokenizer.json"}
+
+      self.vocab_files_names = VOCAB_FILES_NAMES
+      self.model_input_names = ["input_ids", "attention_mask"]
+      # self.slow_tokenizer_class = XLMRobertaTokenizer
+
+      def initialize(
+        vocab_file: nil,
+        tokenizer_file: nil,
+        bos_token: "<s>",
+        eos_token: "</s>",
+        sep_token: "</s>",
+        cls_token: "<s>",
+        unk_token: "<unk>",
+        pad_token: "<pad>",
+        mask_token: "<mask>",
+        **kwargs
+      )
+        # Mask token behave like a normal word, i.e. include the space before it
+        mask_token = mask_token.is_a?(String) ? Tokenizers::AddedToken.new(mask_token, lstrip: true, rstrip: false) : mask_token
+
+        super(vocab_file, tokenizer_file: tokenizer_file, bos_token: bos_token, eos_token: eos_token, sep_token: sep_token, cls_token: cls_token, unk_token: unk_token, pad_token: pad_token, mask_token: mask_token, **kwargs)
+
+        @vocab_file = vocab_file
+      end
+
+      def can_save_slow_tokenizer
+        @vocab_file ? File.exist?(@vocab_file) : false
+      end
+
+      def build_inputs_with_special_tokens(token_ids_0, token_ids_1: nil)
+        if token_ids_1.nil?
+          return [@cls_token_id] + token_ids_0 + [@sep_token_id]
+        end
+        cls = [@cls_token_id]
+        sep = [@sep_token_id]
+        cls + token_ids_0 + sep + sep + token_ids_1 + sep
+      end
+
+      def create_token_type_ids_from_sequences(token_ids_0, token_ids_1: nil)
+        sep = [@sep_token_id]
+        cls = [@cls_token_id]
+
+        if token_ids_1.nil?
+          return (cls + token_ids_0 + sep).length * [0]
+        end
+        (cls + token_ids_0 + sep + sep + token_ids_1 + sep).length * [0]
+      end
+    end
+  end
+end

data/lib/transformers/pipelines/_init.rb

@@ -89,6 +89,16 @@ module Transformers
       },
       "type" => "text"
     },
+    "reranking" => {
+      "impl" => RerankingPipeline,
+      "pt" => [AutoModelForSequenceClassification],
+      "default" => {
+        "model" => {
+          "pt" => ["mixedbread-ai/mxbai-rerank-base-v1", "03241da"]
+        }
+      },
+      "type" => "text"
+    }
   }
 
   PIPELINE_REGISTRY = PipelineRegistry.new(supported_tasks: SUPPORTED_TASKS, task_aliases: TASK_ALIASES)

data/lib/transformers/pipelines/reranking.rb

@@ -0,0 +1,33 @@
+module Transformers
+  class RerankingPipeline < Pipeline
+    def _sanitize_parameters(**kwargs)
+      [{}, {}, kwargs]
+    end
+
+    def preprocess(inputs)
+      @tokenizer.(
+        [inputs[:query]] * inputs[:documents].length,
+        text_pair: inputs[:documents],
+        return_tensors: @framework
+      )
+    end
+
+    def _forward(model_inputs)
+      model_outputs = @model.(**model_inputs)
+      model_outputs
+    end
+
+    def call(query, documents)
+      super({query: query, documents: documents})
+    end
+
+    def postprocess(model_outputs)
+       model_outputs[0]
+        .sigmoid
+        .squeeze
+        .to_a
+        .map.with_index { |s, i| {index: i, score: s} }
+        .sort_by { |v| -v[:score] }
+    end
+  end
+end

data/lib/transformers/version.rb

@@ -1,3 +1,3 @@
 module Transformers
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/lib/transformers.rb

@@ -61,17 +61,32 @@ require_relative "transformers/models/bert/modeling_bert"
 require_relative "transformers/models/bert/tokenization_bert"
 require_relative "transformers/models/bert/tokenization_bert_fast"
 
+# models deberta-v2
+require_relative "transformers/models/deberta_v2/configuration_deberta_v2"
+require_relative "transformers/models/deberta_v2/modeling_deberta_v2"
+require_relative "transformers/models/deberta_v2/tokenization_deberta_v2_fast"
+
 # models distilbert
 require_relative "transformers/models/distilbert/configuration_distilbert"
 require_relative "transformers/models/distilbert/modeling_distilbert"
 require_relative "transformers/models/distilbert/tokenization_distilbert"
 require_relative "transformers/models/distilbert/tokenization_distilbert_fast"
 
+# models mpnet
+require_relative "transformers/models/mpnet/configuration_mpnet"
+require_relative "transformers/models/mpnet/modeling_mpnet"
+require_relative "transformers/models/mpnet/tokenization_mpnet_fast"
+
 # models vit
 require_relative "transformers/models/vit/configuration_vit"
 require_relative "transformers/models/vit/image_processing_vit"
 require_relative "transformers/models/vit/modeling_vit"
 
+# models xml-roberta
+require_relative "transformers/models/xlm_roberta/configuration_xlm_roberta"
+require_relative "transformers/models/xlm_roberta/modeling_xlm_roberta"
+require_relative "transformers/models/xlm_roberta/tokenization_xlm_roberta_fast"
+
 # pipelines
 require_relative "transformers/pipelines/base"
 require_relative "transformers/pipelines/feature_extraction"
@@ -80,6 +95,7 @@ require_relative "transformers/pipelines/image_classification"
 require_relative "transformers/pipelines/image_feature_extraction"
 require_relative "transformers/pipelines/pt_utils"
 require_relative "transformers/pipelines/question_answering"
+require_relative "transformers/pipelines/reranking"
 require_relative "transformers/pipelines/text_classification"
 require_relative "transformers/pipelines/token_classification"
 require_relative "transformers/pipelines/_init"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: transformers-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-10 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.2
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: torch-rb
   requirement: !ruby/object:Gem::Requirement
@@ -104,13 +104,22 @@ files:
 - lib/transformers/models/bert/modeling_bert.rb
 - lib/transformers/models/bert/tokenization_bert.rb
 - lib/transformers/models/bert/tokenization_bert_fast.rb
+- lib/transformers/models/deberta_v2/configuration_deberta_v2.rb
+- lib/transformers/models/deberta_v2/modeling_deberta_v2.rb
+- lib/transformers/models/deberta_v2/tokenization_deberta_v2_fast.rb
 - lib/transformers/models/distilbert/configuration_distilbert.rb
 - lib/transformers/models/distilbert/modeling_distilbert.rb
 - lib/transformers/models/distilbert/tokenization_distilbert.rb
 - lib/transformers/models/distilbert/tokenization_distilbert_fast.rb
+- lib/transformers/models/mpnet/configuration_mpnet.rb
+- lib/transformers/models/mpnet/modeling_mpnet.rb
+- lib/transformers/models/mpnet/tokenization_mpnet_fast.rb
 - lib/transformers/models/vit/configuration_vit.rb
 - lib/transformers/models/vit/image_processing_vit.rb
 - lib/transformers/models/vit/modeling_vit.rb
+- lib/transformers/models/xlm_roberta/configuration_xlm_roberta.rb
+- lib/transformers/models/xlm_roberta/modeling_xlm_roberta.rb
+- lib/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.rb
 - lib/transformers/pipelines/_init.rb
 - lib/transformers/pipelines/base.rb
 - lib/transformers/pipelines/embedding.rb
@@ -119,6 +128,7 @@ files:
 - lib/transformers/pipelines/image_feature_extraction.rb
 - lib/transformers/pipelines/pt_utils.rb
 - lib/transformers/pipelines/question_answering.rb
+- lib/transformers/pipelines/reranking.rb
 - lib/transformers/pipelines/text_classification.rb
 - lib/transformers/pipelines/token_classification.rb
 - lib/transformers/ruby_utils.rb