rbbt-dm 1.1.63 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7939251873b8bf86dccb5260dc9f4b64abb7c24b5001f436a1b457d4ad2333af
-  data.tar.gz: 476b2c7175b557bc287f928225d7da020d8e29a1e59b534d595ea6f75597f23f
+  metadata.gz: ab775c0224960820e5c62e294e6a183be49201da15710b66544762e1aaf97ebf
+  data.tar.gz: 8fffb47ba226f06d1f41a8893d085bdc12c33c021cf2f0152f4cc741db36e420
 SHA512:
-  metadata.gz: bf503009cf5bc8d1ac239f1c8fe07288102560c3bc5324368187690b523f04f92ff94b74bf97c512cb0d4378985f9269177d7dda0897694a0eb2df62f369decc
-  data.tar.gz: 13c8cd26daff6205de91f5c733563fe3d28af056869584882a9eb08d0d1ddcf5bde88bf38645dc82d9f04864631b3dccc93bf1a56945372c3b2a4bad618f3144
+  metadata.gz: 8be084156063cd93c7fe905bc4b6248dd376bbfcff8e650cbb03a4cc5c28f29dbcdaa2801895a3663067d7660e8bc2cf96682829519ebd6511a7a74cec021da0
+  data.tar.gz: 9c8570722319caf5afe60c90778d0b8517e70064030e0081968d919a314cfe1af90f63d13a7b8ce8dd56da6644db89916a729a4005f1e20745c8d4b45d50394c

data/lib/rbbt/vector/model/huggingface.rb

@@ -0,0 +1,140 @@
+require 'rbbt/vector/model'
+require 'rbbt/util/python'
+
+RbbtPython.init_rbbt
+
+class HuggingfaceModel < VectorModel
+
+  attr_accessor :checkpoint, :task, :locate_tokens, :class_labels
+
+  def tsv_dataset(tsv_dataset_file, elements, labels = nil)
+
+    if labels
+      Open.write(tsv_dataset_file) do |ffile|
+        ffile.puts ["label", "text"].flatten * "\t"
+        elements.zip(labels).each do |element,label|
+          ffile.puts [label, element].flatten * "\t"
+        end
+      end
+    else
+      Open.write(tsv_dataset_file) do |ffile|
+        ffile.puts ["text"].flatten * "\t"
+        elements.each{|element| ffile.puts element }
+      end
+    end
+
+    tsv_dataset_file
+  end
+
+  def call_method(name, *args)
+    RbbtPython.import_method("rbbt_dm.huggingface", name).call(*args)
+  end
+
+  def input_tsv_file
+    File.join(@directory, 'dataset.tsv') if @directory
+  end
+
+  def checkpoint_dir
+    File.join(@directory, 'checkpoints') if @directory
+  end
+
+  def run_model(elements, labels = nil)
+    TmpFile.with_file do |tmpfile|
+      tsv_file = input_tsv_file || File.join(tmpfile, 'dataset.tsv')
+      output_dir = checkpoint_dir || File.join(tmpfile, 'checkpoints')
+
+      Open.mkdir File.dirname(output_dir)
+      Open.mkdir File.dirname(tsv_file)
+
+      if labels
+        training_args = call_method(:training_args, output_dir)
+        call_method(:train_model, @model, @tokenizer, training_args, tsv_dataset(tsv_file, elements, labels))
+      else
+        if Array === elements
+          training_args = call_method(:training_args, output_dir)
+          call_method(:predict_model, @model, @tokenizer, training_args, tsv_dataset(tsv_file, elements), @locate_tokens)
+        else
+          call_method(:eval_model, @model, @tokenizer, [elements], @locate_tokens)
+        end
+      end
+    end
+  end
+
+  def initialize(task, initial_checkpoint = nil, *args)
+    super(*args)
+    @task = task
+
+    @checkpoint = model_file && File.exists?(model_file)? model_file : initial_checkpoint
+
+    @model, @tokenizer = call_method(:load_model_and_tokenizer, @task, @checkpoint)
+
+    @locate_tokens = @tokenizer.special_tokens_map["mask_token"]  if @task == "MaskedLM"
+
+    train_model do |file,elements,labels|
+      run_model(elements, labels)
+
+      @model.save_pretrained(file) if file
+      @tokenizer.save_pretrained(file) if file
+    end
+
+    eval_model do |file,elements|
+      run_model(elements)
+    end
+
+    post_process do |result|
+      if result.respond_to?(:predictions)
+        single = false
+        predictions = result.predictions
+      elsif result["token_positions"]
+        predictions = result["result"].predictions
+        token_positions = result["token_positions"]
+      else
+        single = true
+        predictions = result["logits"]
+      end
+
+      result = case @task
+               when "SequenceClassification"
+                 RbbtPython.collect(predictions) do |logits|
+                   logits = RbbtPython.numpy2ruby logits
+                   best_class = logits.index logits.max
+                   best_class = @class_labels[best_class] if @class_labels
+                   best_class
+                 end
+               when "MaskedLM"
+                 all_token_positions = token_positions.to_a
+
+                 i = 0
+                 RbbtPython.collect(predictions) do |item_logits|
+                   item_token_positions = all_token_positions[i]
+                   i += 1
+
+                   item_logits = RbbtPython.numpy2ruby(item_logits)
+                   item_masks = item_token_positions.collect do |token_positions|
+
+                     best = item_logits.values_at(*token_positions).collect do |logits|
+                       best_token, best_score = nil
+                       logits.each_with_index do |v,i|
+                         if best_score.nil? || v > best_score
+                           best_token, best_score = i, v
+                         end
+                       end
+                       best_token
+                     end
+
+                     best.collect{|b| @tokenizer.decode(b) } * "|"
+                   end
+                   Array === @locate_tokens ? item_masks : item_masks.first
+                 end
+               else
+                 logits
+               end
+
+      single ? result.first : result
+    end
+  end
+end
+
+if __FILE__ == $0
+
+end

data/lib/rbbt/vector/model.rb

@@ -2,9 +2,30 @@ require 'rbbt/util/R'
 require 'rbbt/vector/model/util'
 
 class VectorModel
-  attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model
+  attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model, :post_process
   attr_accessor :features, :names, :labels, :factor_levels
 
+  def extract_features(&block)
+    @extract_features = block if block_given?
+    @extract_features
+  end
+
+  def train_model(&block)
+    @train_model = block if block_given?
+    @train_model
+  end
+
+  def eval_model(&block)
+    @eval_model = block if block_given?
+    @eval_model
+  end
+
+  def post_process(&block)
+    @post_process = block if block_given?
+    @post_process
+  end
+
+
   def self.R_run(model_file, features, labels, code, names = nil, factor_levels = nil)
     TmpFile.with_file do |feature_file|
       Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
@@ -101,25 +122,27 @@ cat(paste(label, sep="\\n", collapse="\\n"));
 
   def __load_method(file)
     code = Open.read(file)
-    code.sub!(/.*Proc\.new/, "Proc.new")
+    code.sub!(/.*(\sdo\b|{)/, 'Proc.new\1')
     instance_eval code, file
   end
 
-  def initialize(directory, extract_features = nil, train_model = nil, eval_model = nil, names = nil, factor_levels = nil)
+  def initialize(directory = nil, extract_features = nil, train_model = nil, eval_model = nil, names = nil, factor_levels = nil)
     @directory = directory
-    FileUtils.mkdir_p @directory unless File.exists? @directory
-
-    @model_file = File.join(@directory, "model")
-    @extract_features_file = File.join(@directory, "features")
-    @train_model_file = File.join(@directory, "train_model")
-    @eval_model_file = File.join(@directory, "eval_model")
-    @train_model_file_R = File.join(@directory, "train_model.R")
-    @eval_model_file_R = File.join(@directory, "eval_model.R")
-    @names_file = File.join(@directory, "feature_names")
-    @levels_file = File.join(@directory, "levels")
+    if @directory
+      FileUtils.mkdir_p @directory unless File.exists?(@directory)
+
+      @model_file = File.join(@directory, "model")
+      @extract_features_file = File.join(@directory, "features")
+      @train_model_file = File.join(@directory, "train_model")
+      @eval_model_file = File.join(@directory, "eval_model")
+      @train_model_file_R = File.join(@directory, "train_model.R")
+      @eval_model_file_R = File.join(@directory, "eval_model.R")
+      @names_file = File.join(@directory, "feature_names")
+      @levels_file = File.join(@directory, "levels")
+    end
 
     if extract_features.nil?
-      if File.exists?(@extract_features_file)
+      if @extract_features_file && File.exists?(@extract_features_file)
         @extract_features = __load_method @extract_features_file
       end
     else
@@ -127,9 +150,9 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     end
 
     if train_model.nil?
-      if File.exists?(@train_model_file)
+      if @train_model_file && File.exists?(@train_model_file)
         @train_model = __load_method @train_model_file
-      elsif File.exists?(@train_model_file_R)
+      elsif @train_model_file_R && File.exists?(@train_model_file_R)
         @train_model = Open.read(@train_model_file_R)
       end
     else
@@ -137,9 +160,9 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     end
 
     if eval_model.nil?
-      if File.exists?(@eval_model_file)
+      if @eval_model_file && File.exists?(@eval_model_file)
         @eval_model = __load_method @eval_model_file
-      elsif File.exists?(@eval_model_file_R)
+      elsif @eval_model_file_R && File.exists?(@eval_model_file_R)
         @eval_model = Open.read(@eval_model_file_R)
       end
     else
@@ -147,7 +170,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     end
 
     if names.nil?
-      if File.exists?(@names_file)
+      if @names_file && File.exists?(@names_file)
         @names = Open.read(@names_file).split("\n")
       end
     else
@@ -155,10 +178,10 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     end
 
     if factor_levels.nil?
-      if File.exists?(@levels_file)
+      if @levels_file && File.exists?(@levels_file)
         @factor_levels = YAML.load(Open.read(@levels_file))
       end
-      if File.exists?(@model_file + '.factor_levels')
+      if @model_file && File.exists?(@model_file + '.factor_levels')
         @factor_levels = TSV.open(@model_file + '.factor_levels')
       end
     else
@@ -175,7 +198,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
   end
 
   def add(element, label = nil)
-    features = @extract_features ? extract_features.call(element) : element
+    features = @extract_features ? self.instance_exec(element, &@extract_features) : element
     @features << features
     @labels << label 
   end
@@ -186,7 +209,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
         add(elem, label)
       end
     else
-      features = @extract_features.call(nil, elements)
+      features = self.instance_exec(nil, elements, &@extract_features)
       @features.concat  features
       @labels.concat labels if labels
     end
@@ -223,9 +246,9 @@ cat(paste(label, sep="\\n", collapse="\\n"));
 
   def train
     case 
-    when Proc === train_model
-      train_model.call(@model_file, @features, @labels, @names, @factor_levels)
-    when String === train_model
+    when Proc === @train_model
+      self.instance_exec(@model_file, @features, @labels, @names, @factor_levels, &@train_model)
+    when String === @train_model
       VectorModel.R_train(@model_file,  @features, @labels, train_model, @names, @factor_levels)
     end
     save_models
@@ -236,32 +259,44 @@ cat(paste(label, sep="\\n", collapse="\\n"));
   end
 
   def eval(element)
-    case 
-    when Proc === @eval_model
-      @eval_model.call(@model_file, @extract_features.call(element), false, nil, @names, @factor_levels)
-    when String === @eval_model
-      VectorModel.R_eval(@model_file,  @extract_features.call(element), false, eval_model, @names, @factor_levels)
-    end
+    features = @extract_features.nil? ? element : self.instance_exec(element, &@extract_features)
+
+    result = case 
+             when Proc === @eval_model
+               self.instance_exec(@model_file, features, false, nil, @names, @factor_levels, &@eval_model)
+             when String === @eval_model
+               VectorModel.R_eval(@model_file, features, false, eval_model, @names, @factor_levels)
+             else
+               raise "No @eval_model function or R script"
+             end
+
+    result = self.instance_exec(result, &@post_process) if Proc === @post_process 
+
+    result
   end
 
   def eval_list(elements, extract = true)
 
     if extract && ! @extract_features.nil? 
       features = if @extract_features.arity == 1
-                   elements.collect{|element| @extract_features.call(element) }
+                   elements.collect{|element| self.instance_exec(element, &@extract_features) }
                  else
-                   @extract_features.call(nil, elements)
+                   self.instance_exec(nil, elements, &@extract_features)
                  end
     else
       features = elements
     end
 
-    case 
-    when Proc === eval_model
-      eval_model.call(@model_file, features, true, nil, @names, @factor_levels)
-    when String === eval_model
-      VectorModel.R_eval(@model_file, features, true, eval_model, @names, @factor_levels)
-    end
+    result = case 
+             when Proc === eval_model
+               self.instance_exec(@model_file, features, true, nil, @names, @factor_levels, &@eval_model)
+             when String === eval_model
+               VectorModel.R_eval(@model_file, features, true, eval_model, @names, @factor_levels)
+             end
+
+    result = self.instance_exec(result, &@post_process) if Proc === @post_process 
+
+    result
   end
 
   #def cross_validation(folds = 10)

data/test/rbbt/vector/model/test_huggingface.rb

@@ -0,0 +1,116 @@
+require File.join(File.expand_path(File.dirname(__FILE__)),'../../..', 'test_helper.rb')
+require 'rbbt/vector/model/huggingface'
+
+class TestHuggingface < Test::Unit::TestCase
+
+  def test_sst_eval
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+
+      model.class_labels = ["Bad", "Good"]
+
+      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+
+    end
+  end
+
+
+  def test_sst_train
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.class_labels = ["Bad", "Good"]
+
+      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+
+      100.times do
+        model.add "Dog is good", 1
+      end
+
+      model.train
+
+      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+    end
+  end
+
+  def test_sst_train_no_save
+    checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+    model = HuggingfaceModel.new "SequenceClassification", checkpoint
+    model.class_labels = ["Bad", "Good"]
+
+    assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+
+    100.times do
+      model.add "Dog is good", 1
+    end
+
+    model.train
+
+    assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+  end
+
+  def test_sst_train_save_and_load
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.class_labels = ["Bad", "Good"]
+
+      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+
+      100.times do
+        model.add "Dog is good", 1
+      end
+
+      model.train
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.class_labels = ["Bad", "Good"]
+
+      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+
+      model = HuggingfaceModel.new "SequenceClassification", model.model_file
+      model.class_labels = ["Bad", "Good"]
+
+      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+
+    end
+  end
+
+  def test_sst_stress_test
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+
+      100.times do
+        model.add "Dog is good", 1
+        model.add "Cat is bad", 0
+      end
+
+      Misc.benchmark(10) do
+        model.train
+      end
+
+      Misc.benchmark 1000 do
+        model.eval(["This is good", "This is terrible", "This is dog", "This is cat", "Very different stuff", "Dog is bad", "Cat is good"])
+      end
+    end
+  end
+
+  def test_mask_eval
+    checkpoint = "bert-base-uncased"
+
+    model = HuggingfaceModel.new "MaskedLM", checkpoint
+    assert_equal 3, model.eval(["Paris is the [MASK] of the France.", "The [MASK] worked very hard all the time.", "The [MASK] arrested the dangerous [MASK]."]).
+      reject{|v| v.empty?}.length
+
+  end
+
+
+end
+

data/test/rbbt/vector/test_model.rb

@@ -282,6 +282,7 @@ cat(label, file="#{results}");
         model.add features, label
       end
 
+      iii model.eval("1;1;1")
       assert model.eval("1;1;1").to_f > 0.5
       assert model.eval("0;0;0").to_f < 0.5
     end
@@ -509,5 +510,28 @@ label = predict(model, features);
     end
   end
 
+  def test_python
+    require 'rbbt/util/python'
+    TmpFile.with_file do |dir|
+      model = VectorModel.new dir
+
+      model.eval_model do |file, elements|
+        elements = [elements] unless Array === elements
+        RbbtPython.binding_run  do
+          pyimport :torch
+          rand = torch.rand(1).numpy[0].to_f
+          elements.collect{|e| e >= rand ? 1 : 0 }
+        end
+      end
+      p1, p2 = model.eval [0.9, 0.1]
+      assert p2 <= p1
+
+      model = VectorModel.new dir
+      assert p2 <= p1
+
+    end
+
+  end
+
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.63
+  version: 1.2.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-15 00:00:00.000000000 Z
+date: 2023-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -107,6 +107,7 @@ files:
 - lib/rbbt/statistics/rank_product.rb
 - lib/rbbt/tensorflow.rb
 - lib/rbbt/vector/model.rb
+- lib/rbbt/vector/model/huggingface.rb
 - lib/rbbt/vector/model/random_forest.rb
 - lib/rbbt/vector/model/spaCy.rb
 - lib/rbbt/vector/model/svm.rb
@@ -131,6 +132,7 @@ files:
 - test/rbbt/statistics/test_random_walk.rb
 - test/rbbt/test_ml_task.rb
 - test/rbbt/test_stan.rb
+- test/rbbt/vector/model/test_huggingface.rb
 - test/rbbt/vector/model/test_spaCy.rb
 - test/rbbt/vector/model/test_svm.rb
 - test/rbbt/vector/model/test_tensorflow.rb
@@ -154,21 +156,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Data-mining and statistics
 test_files:
-- test/rbbt/network/test_paths.rb
-- test/rbbt/matrix/test_barcode.rb
+- test/test_helper.rb
+- test/rbbt/vector/test_model.rb
+- test/rbbt/vector/model/test_huggingface.rb
+- test/rbbt/vector/model/test_tensorflow.rb
+- test/rbbt/vector/model/test_spaCy.rb
+- test/rbbt/vector/model/test_svm.rb
 - test/rbbt/statistics/test_random_walk.rb
 - test/rbbt/statistics/test_fisher.rb
 - test/rbbt/statistics/test_fdr.rb
 - test/rbbt/statistics/test_hypergeometric.rb
-- test/rbbt/test_ml_task.rb
-- test/rbbt/vector/test_model.rb
-- test/rbbt/vector/model/test_spaCy.rb
-- test/rbbt/vector/model/test_tensorflow.rb
-- test/rbbt/vector/model/test_svm.rb
 - test/rbbt/test_stan.rb
-- test/test_helper.rb
+- test/rbbt/matrix/test_barcode.rb
+- test/rbbt/test_ml_task.rb
+- test/rbbt/network/test_paths.rb