rbbt-dm 1.2.6 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9744ab9faeaf4f9cc04947eb11103dbf0694dda624f805a5c6be27bb22af81ce
-  data.tar.gz: d3a3903aa276a69e20cbd71213286449db396ecf5f6a4b4d80a64ab299041fbb
+  metadata.gz: 1c55843bf543c88167239f6e182495963e0683c5a7fdd7c3a7ab9bd501a78bc8
+  data.tar.gz: d01aaf45331766eac6d868749b8df72c49d1a6888f44f7a1d4f8cbfefe258c87
 SHA512:
-  metadata.gz: 263fb609b37522874426bcd79374760399b4a9aaab443ae6d74c727f2d148474dd71ee0b2cfda7a50131dafbc314f66352f4285562a75f62144d2e05ccd214c7
-  data.tar.gz: 1e0426429a38028a19b3f8c955e975138199c791dad8691de7fb760a5cbec3304f19341906a4457563a90de965ee2f12a5a639b928866b46258af2507eeb39fa
+  metadata.gz: 7b6a225ce0403759ab45f26d371d491c19fc76f6560771868a58b9de921fd3aa03750bd7aec95c34029f61f53e71e382958f2779ca790fde30958cfbd1169a0b
+  data.tar.gz: ae1b6d44072398fbde96a0cb31f9586076dee1a5c7e2ac32726c65ecaaa3d08b59ea627c7a0f9f4a8e87547d5a403452ea5bee1d0736d610bf73b6456cb99be9

data/lib/rbbt/vector/model/spaCy.rb

@@ -75,14 +75,6 @@ class SpaCyModel < VectorModel
             d.cats.sort_by{|l,v| v.to_f || 0 }.last.first
           end
         end
-        #nlp.(docs).cats.collect{|cats| cats.sort_by{|l,v| v.to_f }.last.first }
-        #Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
-        #  texts.collect do |text|
-        #    cats = nlp.(text).cats
-        #    bar.tick
-        #    cats.sort_by{|l,v| v.to_f }.last.first
-        #  end
-        #end
       end
     end
   end

data/lib/rbbt/vector/model/util.rb

@@ -9,4 +9,22 @@ class VectorModel
     @bar.init
     @bar
   end
+
+  def balance_labels
+    counts = Misc.counts(@labels)
+    min = counts.values.min
+
+    used = {}
+    new_labels = []
+    new_features = []
+    @labels.zip(@features).shuffle.each do |label, features|
+      used[label] ||= 0
+      next if used[label] > min
+      used[label] += 1
+      new_labels << label
+      new_features << features
+    end
+    @labels = new_labels
+    @features = new_features
+  end
 end

data/lib/rbbt/vector/model.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/R'
 require 'rbbt/vector/model/util'
 
 class VectorModel
-  attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model, :post_process
+  attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model, :post_process, :balance
   attr_accessor :features, :names, :labels, :factor_levels
   attr_accessor :model_options
 
@@ -270,19 +270,32 @@ cat(paste(label, sep="\\n", collapse="\\n"));
       Open.write(@post_process_file_R, post_process)
     end
 
-
     Open.write(@levels_file, @factor_levels.to_yaml) if @factor_levels
     Open.write(@names_file, @names * "\n" + "\n") if @names
     Open.write(@options_file, @model_options.to_json) if @model_options
   end
 
   def train
-    case 
-    when Proc === @train_model
-      self.instance_exec(@model_file, @features, @labels, @names, @factor_levels, &@train_model)
-    when String === @train_model
-      VectorModel.R_train(@model_file,  @features, @labels, train_model, @names, @factor_levels)
+    begin
+      if @balance
+        @original_features = @features
+        @original_labels = @labels
+        self.balance_labels
+      end
+
+      case 
+      when Proc === @train_model
+        self.instance_exec(@model_file, @features, @labels, @names, @factor_levels, &@train_model)
+      when String === @train_model
+        VectorModel.R_train(@model_file,  @features, @labels, train_model, @names, @factor_levels)
+      end
+    ensure
+      if @balance
+        @features =  @original_features
+        @labels = @original_labels
+      end
     end
+
     save_models if @directory
   end
 
@@ -438,6 +451,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
       @features = orig_features
       @labels = orig_labels
     end unless folds == -1
+
     self.reset_model if self.respond_to? :reset_model
     self.train unless folds == 1
     res

data/python/rbbt_dm/huggingface.py

@@ -53,7 +53,7 @@ def load_tsv(tsv_file):
 
 def tsv_dataset(tokenizer, tsv_file):
     dataset = load_tsv(tsv_file)
-    tokenized_dataset = dataset.map(lambda example: tokenizer(example["text"], truncation=True) , batched=True)
+    tokenized_dataset = dataset.map(lambda example: tokenizer(example["text"], truncation=True, max_length=512) , batched=True)
     return tokenized_dataset
 
 def training_args(*args, **kwargs):

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.7
 platform: ruby
 authors:
 - Miguel Vazquez