rbbt-dm 1.2.7 → 1.2.10

data/test/rbbt/vector/model/test_huggingface.rb

@@ -3,7 +3,7 @@ require 'rbbt/vector/model/huggingface'
 
 class TestHuggingface < Test::Unit::TestCase
 
-  def test_options
+  def _test_options
     TmpFile.with_file do |dir|
       checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
       task = "SequenceClassification"
@@ -11,20 +11,20 @@ class TestHuggingface < Test::Unit::TestCase
       model = HuggingfaceModel.new task, checkpoint, dir, :class_labels => %w(bad good)
       iii model.eval "This is dog"
       iii model.eval "This is cat"
-      iii model.eval(["This is dog", "This is cat"])
+      iii model.eval_list(["This is dog", "This is cat"])
 
       model = VectorModel.new dir
-      iii model.eval(["This is dog", "This is cat"])
+      iii model.eval_list(["This is dog", "This is cat"])
     end
   end
 
-  def test_pipeline
+  def _test_pipeline
     require 'rbbt/util/python'
     model = VectorModel.new
     model.post_process do |elements|
       elements.collect{|e| e['label'] }
     end
-    model.eval_model do |file, elements|
+    model.eval_model do |elements|
       RbbtPython.run :transformers do 
         classifier ||= transformers.pipeline("sentiment-analysis")
         classifier.call(elements)
@@ -33,21 +33,53 @@ class TestHuggingface < Test::Unit::TestCase
 
     assert_equal ["POSITIVE"], model.eval("I've been waiting for a HuggingFace course my whole life.")
   end
+  
+  def _test_tokenizer_size
+    checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+    tokenizer = RbbtPython.call_method("rbbt_dm.huggingface", :load_tokenizer, 
+                                       "MaskedLM", checkpoint, :max_length => 5, :model_max_length => 5)
+    assert_equal 5, tokenizer.call("This is a sentence that has several words", truncation: true, max_length: 5)["input_ids"].__len__
+    assert_equal 5, tokenizer.call("This is a sentence that has several words", truncation: true)["input_ids"].__len__
+  end
 
   def test_sst_eval
     TmpFile.with_file do |dir|
       checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
 
-      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir, :tokenizer_args => {:max_length => 16}
 
       model.model_options[:class_labels] = ["Bad", "Good"]
 
-      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal "Bad", model.eval("This is dog")
+      assert_equal ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"])
     end
   end
 
 
-  def test_sst_train
+  def _test_sst_train
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir, max_length: 128
+
+      model.model_options[:class_labels] = %w(Bad Good)
+
+      assert_equal ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"])
+
+      100.times do
+        model.add "Dog is good", "Good"
+      end
+
+      model.train
+
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
+
+      model = VectorModel.new dir
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
+    end
+  end
+
+  def _test_sst_train_with_labels
     TmpFile.with_file do |dir|
       checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
 
@@ -55,28 +87,29 @@ class TestHuggingface < Test::Unit::TestCase
 
       model.model_options[:class_labels] = %w(Bad Good)
 
-      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"])
 
       100.times do
-        model.add "Dog is good", 1
+        model.add "Dog is good", "Good"
       end
 
       model.train
 
-      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
 
       model = VectorModel.new dir
-      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
     end
   end
 
-  def test_sst_train_no_save
+
+  def _test_sst_train_no_save
     checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
 
     model = HuggingfaceModel.new "SequenceClassification", checkpoint
     model.model_options[:class_labels] = ["Bad", "Good"]
 
-    assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+    assert_equal ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"])
 
     100.times do
       model.add "Dog is good", 1
@@ -84,48 +117,50 @@ class TestHuggingface < Test::Unit::TestCase
 
     model.train
 
-    assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+    assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
   end
 
-  def test_sst_train_save_and_load
+  def _test_sst_train_save_and_load
     TmpFile.with_file do |dir|
       checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
 
       model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
       model.model_options[:class_labels] = ["Bad", "Good"]
 
-      assert_equal ["Bad", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"])
 
       100.times do
-        model.add "Dog is good", 1
+        model.add "Dog is good", "Good"
       end
 
       model.train
 
       model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
 
-      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
 
-      model_file = model.model_file
+      model_path = model.model_path
 
-      model = HuggingfaceModel.new "SequenceClassification", model_file
+      model = HuggingfaceModel.new "SequenceClassification", model_path
       model.model_options[:class_labels] = ["Bad", "Good"]
 
-      assert_equal ["Good", "Good"], model.eval(["This is dog", "This is cat"])
+      assert_equal ["Good", "Good"], model.eval_list(["This is dog", "This is cat"])
 
       model = VectorModel.new dir
 
-      assert_equal "Good", model.eval("This is dog")
+      assert_equal "Good", model.eval_list("This is dog")
 
     end
   end
 
-  def test_sst_stress_test
+  def _test_sst_stress_test
     TmpFile.with_file do |dir|
       checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
 
       model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
 
+      assert_equal 0, model.eval("This is dog")
+
       100.times do
         model.add "Dog is good", 1
         model.add "Cat is bad", 0
@@ -136,18 +171,214 @@ class TestHuggingface < Test::Unit::TestCase
       end
 
       Misc.benchmark 1000 do
-        model.eval(["This is good", "This is terrible", "This is dog", "This is cat", "Very different stuff", "Dog is bad", "Cat is good"])
+        model.eval_list(["This is good", "This is terrible", "This is dog", "This is cat", "Very different stuff", "Dog is bad", "Cat is good"])
       end
     end
   end
 
-  def test_mask_eval
+  def _test_mask_eval
     checkpoint = "bert-base-uncased"
 
     model = HuggingfaceModel.new "MaskedLM", checkpoint
-    assert_equal 3, model.eval(["Paris is the [MASK] of the France.", "The [MASK] worked very hard all the time.", "The [MASK] arrested the dangerous [MASK]."]).
+    assert_equal 3, model.eval_list(["Paris is the [MASK] of the France.", "The [MASK] worked very hard all the time.", "The [MASK] arrested the dangerous [MASK]."]).
       reject{|v| v.empty?}.length
   end
 
+  def _test_mask_eval_tokenizer
+    checkpoint = "bert-base-uncased"
+
+    model = HuggingfaceModel.new "MaskedLM", checkpoint
+
+    mod, tokenizer = model.init
+
+    orig =  tokenizer.call("Hi [GENE]")["input_ids"]
+    tokenizer.add_tokens(["[GENE]"])
+    mod.resize_token_embeddings(tokenizer.__len__)
+    new =  tokenizer.call("Hi [GENE]")["input_ids"]
+
+    assert orig.length > new.length
+  end
+
+
+  def _test_custom_class
+    TmpFile.with_file do |dir|
+      Open.write File.join(dir, "mypkg/__init__.py"), ""
+
+      Open.write File.join(dir, "mypkg/mymodel.py"), <<~EOF
+
+# Esta clase es igual que la de RobertaForTokenClassification
+# Importamos los métodos necesarios
+import torch.nn as nn
+from transformers import RobertaConfig
+from transformers.modeling_outputs import TokenClassifierOutput
+from transformers.models.roberta.modeling_roberta import RobertaModel, RobertaPreTrainedModel
+
+# Creamos una clase que herede de RobertaPreTrainedModel
+class RobertaForTokenClassification_NER(RobertaPreTrainedModel):
+  config_class = RobertaConfig
+
+  def __init__(self, config):
+    # Se usa para inicializar el modelo Roberta
+    super().__init__(config)
+    # Numero de etiquetas que se van a clasificar (sería el número de etiquetas del corpus*2)
+    # Una correspondiente a la etiqueta I y otra a la B.
+    self.num_labels = config.num_labels
+    # No incorporamos pooling layer para devolver los hidden states de cada token (no sólo el CLS)
+    self.roberta = RobertaModel(config, add_pooling_layer=False)
+    self.dropout = nn.Dropout(config.hidden_dropout_prob)
+    self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+    self.init_weights()
+
+  def forward(self, input_ids = None, attention_mask = None, token_type_ids = None, labels = None, 
+              **kwargs):
+    # Obtenemos una codificación del input (los hidden states)
+    outputs = self.roberta(input_ids, attention_mask = attention_mask,
+                           token_type_ids = token_type_ids, **kwargs)
+    
+    # A la salida de los hidden states le aplicamos la capa de dropout
+    sequence_output = self.dropout(outputs[0])
+    # Y posteriormente la capa de clasificación.
+    logits = self.classifier(sequence_output)
+    # Si labels tiene algún valor (lo que se hará durante el proceso de entrenamiento), se calculan las Loss
+    # para justar los pesos en el backprop.
+    loss = None
+    if labels is not None:
+      loss_fct = nn.CrossEntropyLoss()
+      loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+    return TokenClassifierOutput(loss=loss, logits=logits,
+                                 hidden_states=outputs.hidden_states, 
+                                 attentions=outputs.attentions)
+      EOF
+
+      RbbtPython.add_path dir
+
+      biomedical_roberta = "PlanTL-GOB-ES/bsc-bio-ehr-es-cantemist"
+      model = HuggingfaceModel.new "mypkg.mymodel:RobertaForTokenClassification_NER", biomedical_roberta
+
+      model.post_process do |result,is_list|
+        if is_list
+          RbbtPython.numpy2ruby result.predictions
+        else
+          result["logits"][0]
+        end
+      end
+
+      texto = "El paciente tiene un cáncer del pulmon"
+      assert model.eval(texto)[5][1] > 0
+    end
+  end
+
+  def _test_sst_train_word_embeddings
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.model_options[:class_labels] = %w(Bad Good)
+
+      mod, tokenizer = model.init
+
+      orig = HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings')
+      orig = RbbtPython.numpy2ruby(orig.cpu.detach.numpy)
+
+      100.times do
+        model.add "Dog is good", "Good"
+      end
+
+      model.train
+
+      new = HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings')
+      new = RbbtPython.numpy2ruby(new.cpu.detach.numpy)
+      
+      diff = []
+      new.each_with_index do |row,i|
+        diff << i if row != orig[i]
+      end
+
+      assert diff.length > 0
+    end
+  end
+
+  def _test_sst_freeze_word_embeddings
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.model_options[:class_labels] = %w(Bad Good)
+
+      mod, tokenizer = model.init
+
+      layer = HuggingfaceModel.freeze_layer(mod, 'distilbert')
+
+      orig = HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings')
+      orig = RbbtPython.numpy2ruby(orig.cpu.detach.numpy)
+
+      100.times do
+        model.add "Dog is good", "Good"
+      end
+
+      model.train
+
+      new = HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings')
+      new = RbbtPython.numpy2ruby(new.cpu.detach.numpy)
+      
+      diff = []
+      new.each_with_index do |row,i|
+        diff << i if row != orig[i]
+      end
+
+      assert diff.length == 0
+    end
+  end
+
+  def _test_sst_save_word_embeddings
+    TmpFile.with_file do |dir|
+      checkpoint = "distilbert-base-uncased-finetuned-sst-2-english"
+
+      model = HuggingfaceModel.new "SequenceClassification", checkpoint, dir
+      model.model_options[:class_labels] = %w(Bad Good)
+
+      mod, tokenizer = model.init
+
+      100.times do
+        model.add "Dog is good", "Good"
+      end
+
+      model.train
+
+      orig = RbbtPython.numpy2ruby(
+        HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings').cpu.detach.numpy)
+
+      model = HuggingfaceModel.new "MaskedLM", checkpoint, dir
+
+      mod, tokenizer = model.init
+
+      new = RbbtPython.numpy2ruby(
+        HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings').cpu.detach.numpy)
+      
+      
+      diff = []
+      new.each_with_index do |row,i|
+        diff << i if row != orig[i]
+      end
+
+      assert diff.length == 0
+
+      model = HuggingfaceModel.new "MaskedLM", checkpoint
+
+      mod, tokenizer = model.init
+
+      new = RbbtPython.numpy2ruby(
+        HuggingfaceModel.get_weights(mod, 'distilbert.embeddings.word_embeddings').cpu.detach.numpy)
+      
+      
+      diff = []
+      new.each_with_index do |row,i|
+        diff << i if row != orig[i]
+      end
+
+      assert diff.length > 0 
+    end
+  end
 end
 

data/test/rbbt/vector/model/test_python.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestPythonModel < Test::Unit::TestCase
+  def test_linear
+    model = nil
+
+    TmpFile.with_dir do |dir|
+
+      Misc.in_dir dir do
+        Open.write 'model.py', <<-EOF
+class TestModel:
+  def __init__(self, delta):
+    self.delta = delta
+    
+  def eval(self, x):
+    return [e + self.delta for e in x]
+        EOF
+        model = PythonModel.new dir, 'TestModel', :model, delta: 1
+        
+        assert_equal 2, model.eval(1)
+        assert_equal [4, 6], model.eval_list([3, 5])
+
+        model = PythonModel.new dir, 'TestModel', :model, delta: 2
+        
+        assert_equal 3, model.eval(1)
+      end
+    end
+  end
+end
+

data/test/rbbt/vector/model/test_pytorch_lightning.rb

@@ -0,0 +1,97 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/vector/model/pytorch_lightning'
+
+class TestPytorchLightning < Test::Unit::TestCase
+  def test_regresion
+    points = 10
+    a = 1
+    b = 1
+
+    x = (0..points - 1)
+    y = points.times.collect{|p| p }
+    
+    python = <<~EOF
+import pytorch_lightning as pl
+import numpy as np
+import torch
+from torch.nn import MSELoss
+from torch.optim import Adam
+from torch.utils.data import DataLoader, Dataset
+import torch.nn as nn
+
+
+class SimpleDataset(Dataset):
+    def __init__(self):
+        X = np.arange(10000)
+        y = X * 2
+        X = [[_] for _ in X]
+        y = [[_] for _ in y]
+        self.X = torch.Tensor(X)
+        self.y = torch.Tensor(y)
+
+    def __len__(self):
+        return len(self.y)
+
+    def __getitem__(self, idx):
+        return {"X": self.X[idx], "y": self.y[idx]}
+
+
+class TestPytorchLightningModel(pl.LightningModule):
+    def __init__(self):
+        super().__init__()
+        self.fc = nn.Linear(1, 1)
+        self.criterion = MSELoss()
+
+    def forward(self, inputs, labels=None):
+        outputs = self.fc(inputs)
+        loss = 0
+        if labels is not None:
+            loss = self.criterion(outputs, labels)
+        return loss, outputs
+
+    def train_dataloader(self):
+        dataset = SimpleDataset()
+        return DataLoader(dataset, batch_size=1000)
+
+    def training_step(self, batch, batch_idx):
+        input_ids = batch["X"]
+        labels = batch["y"]
+        loss, outputs = self(input_ids, labels)
+        return {"loss": loss}
+
+    def configure_optimizers(self):
+        optimizer = Adam(self.parameters(), lr=0.1)
+        return optimizer
+    EOF
+
+    TmpFile.with_dir do |dir|
+      Open.write(File.join(dir, 'model.py'), python)
+      model = PytorchLightningModel.new dir, "TestPytorchLightningModel"
+      model.init
+
+      model.trainer = RbbtPython.class_new_obj("pytorch_lightning", "Trainer", max_epochs: 10, precision: 16)
+      model.init
+
+      model.train
+
+      w = model.get_weights('fc').to_ruby.first.first
+
+      assert w > 1.8
+      assert w < 2.2
+
+      res = model.eval(10.0)
+      assert_equal res, (10 * w)
+      assert res > 1.8 * 10.0
+      assert res < 2.2 * 10.0
+
+      res = model.eval([10.0])
+      res = model.eval_list([[10.0], [11.2], [14.3]])
+      assert_equal 3, RbbtPython.numpy2ruby(res).length
+
+      model = VectorModel.new dir
+      model.init
+
+    end
+  end
+end
+

data/test/rbbt/vector/model/test_spaCy.rb

@@ -100,7 +100,7 @@ class TestSpaCyModel < Test::Unit::TestCase
       )
 
 
-      Rbbt::Config.set 'gpu_id', nil, :spacy
+      Rbbt::Config.set 'gpu_id', 0, :spacy
       require 'rbbt/tsv/csv'
       url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
       tsv = TSV.csv(Open.open(url))

data/test/rbbt/vector/model/test_tensorflow.rb

@@ -1,5 +1,6 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/vector/model/tensorflow'
+require 'rbbt/util/python'
 
 class TestTensorflowModel < Test::Unit::TestCase
 
@@ -10,6 +11,7 @@ class TestTensorflowModel < Test::Unit::TestCase
 
       model = TensorFlowModel.new(
         dir, 
+        jit_compile: true,
         optimizer: 'adam',
         loss: 'sparse_categorical_crossentropy',
         metrics: ['accuracy']
@@ -53,5 +55,6 @@ class TestTensorflowModel < Test::Unit::TestCase
       assert sum.to_f / predictions.length > 0.7
     end
   end
+
 end
 

data/test/rbbt/vector/model/test_torch.rb

@@ -0,0 +1,61 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestTorch < Test::Unit::TestCase
+  def test_linear
+    model = nil
+
+    TmpFile.with_dir do |dir|
+
+      # Create model
+      
+      model = TorchModel.new dir
+      model.model = RbbtPython.torch.nn.Linear.new(1, 1)
+      model.criterion = RbbtPython.torch.nn.MSELoss.new()
+
+      model.extract_features do |f|
+        [f]
+      end
+
+      model.post_process do |v,list|
+        list ? v.to_ruby.collect{|vv| vv.first } :  v.to_ruby.first
+      end
+
+      # Train model
+      
+      model.add 5.0, [10.0]
+      model.add 10.0, [20.0]
+
+      model.training_args[:epochs] = 1000
+      model.train
+
+      w = model.get_weights.to_ruby.first.first
+
+      assert w > 1.8
+      assert w < 2.2
+
+      # Load the model again
+
+      model = VectorModel.new dir
+
+      # Test model
+
+      y = model.eval(100.0)
+
+      assert(y > 150.0)
+      assert(y < 250.0)
+
+      test = [1.0, 5.0, 10.0, 20.0]
+      input_sum = Misc.sum(test)
+      sum = Misc.sum(model.eval_list(test))
+      assert sum > 0.8 * input_sum * 2
+      assert sum < 1.2 * input_sum * 2
+
+      w = TorchModel.get_weights(model.model).to_ruby.first.first
+
+      assert w > 1.8
+      assert w < 2.2
+    end
+  end
+end
+