rbbt-dm 1.2.6 → 1.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9744ab9faeaf4f9cc04947eb11103dbf0694dda624f805a5c6be27bb22af81ce
-  data.tar.gz: d3a3903aa276a69e20cbd71213286449db396ecf5f6a4b4d80a64ab299041fbb
+  metadata.gz: db2cbab94e21fd2ca67f7306fa9941b59cbfb2865382e5439edf6313f50309e7
+  data.tar.gz: f4acf3651daa90ef23bc454c62df68e208976a977d51e2e85d02558d48897187
 SHA512:
-  metadata.gz: 263fb609b37522874426bcd79374760399b4a9aaab443ae6d74c727f2d148474dd71ee0b2cfda7a50131dafbc314f66352f4285562a75f62144d2e05ccd214c7
-  data.tar.gz: 1e0426429a38028a19b3f8c955e975138199c791dad8691de7fb760a5cbec3304f19341906a4457563a90de965ee2f12a5a639b928866b46258af2507eeb39fa
+  metadata.gz: 7786759636450821aabca306cd210ab3d201e094b81bb70052d57d7bfb6e4de73a198576fe4b002487baf7997138f9c53b91644632cd12cc79b40ff62141a70a
+  data.tar.gz: 9870745068a897909170f3a6187e520e6530b121f8ad4ab40224c3369a16a8bb1c1e55bb8ca8fd892943ec1fcead9f4661e49c3d97d07d3740236a1ec4f69a34

data/lib/rbbt/matrix/barcode.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/R'
 class RbbtMatrix
   def barcode(outfile, factor = 2)
 
-    FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exists? File.dirname(outfile)
+    FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exist? File.dirname(outfile)
     cmd =<<-EOF
 source('#{Rbbt.share.R['barcode.R'].find}')
 rbbt.GE.barcode.mode(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{ R.ruby2R factor })
@@ -49,7 +49,7 @@ rbbt.GE.barcode.mode(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{ R.r
 
     clusters = Array === clusters ? clusters : (2..clusters).to_a
 
-    FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exists? File.dirname(outfile)
+    FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exist? File.dirname(outfile)
     cmd =<<-EOF
 source('#{Rbbt.share.R['barcode.R'].find}')
 rbbt.GE.activity_cluster(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{R.ruby2R key_field}, #{R.ruby2R clusters})

data/lib/rbbt/matrix/differential.rb

@@ -45,7 +45,7 @@ class RbbtMatrix
       end
 
       file = file.find if Path === file
-      FileUtils.mkdir_p File.dirname(file) unless file.nil? or File.exists? File.dirname(file)
+      FileUtils.mkdir_p File.dirname(file) unless file.nil? or File.exist? File.dirname(file)
 
       cmd = <<-EOS
 
@@ -71,10 +71,10 @@ end
 
 
 #def self.analyze(datafile,  main, contrast = nil, log2 = false, outfile = nil, key_field = nil, two_channel = nil)
-#  FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exists? File.dirname(outfile)
+#  FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exist? File.dirname(outfile)
 #  GE.run_R("rbbt.GE.process(#{ R.ruby2R datafile }, main = #{R.ruby2R(main, :strings => true)}, contrast = #{R.ruby2R(contrast, :strings => true)}, log2=#{ R.ruby2R log2 }, outfile = #{R.ruby2R outfile}, key.field = #{R.ruby2R key_field}, two.channel = #{R.ruby2R two_channel})")
 #end
 #def self.barcode(datafile, outfile, factor = 2)
-#  FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exists? File.dirname(outfile)
+#  FileUtils.mkdir_p File.dirname(outfile) unless outfile.nil? or File.exist? File.dirname(outfile)
 #  GE.run_R("rbbt.GE.barcode(#{ R.ruby2R datafile }, #{ R.ruby2R outfile }, #{ R.ruby2R factor })")
 #end

data/lib/rbbt/matrix/knowledge_base.rb

@@ -13,7 +13,7 @@ class KnowledgeBase
 
     return matrix if RbbtMatrix === matrix
 
-    Path.setup(matrix) if not Path === matrix and File.exists? matrix
+    Path.setup(matrix) if not Path === matrix and File.exist? matrix
 
     raise "Registered matrix is strange: #{Misc.fingerprint matrix}" unless Path === matrix
 

data/lib/rbbt/plots/bar.rb

@@ -119,7 +119,7 @@ module BarPlot
     options = Misc.add_defaults options, :width => [options[:total], MAX_WIDTH].min, :height => 20, :background => PNG::Color::White
     width, height, background, canvas = Misc.process_options options, :width, :height, :background, :canvas
 
-    canvas ||= if options[:update] and options[:filename] and File.exists? options[:filename]
+    canvas ||= if options[:update] and options[:filename] and File.exist? options[:filename]
                  PNG.load_file options[:filename]
                else
                  PNG::Canvas.new width, height, get_color(background) 

data/lib/rbbt/stan.rb

@@ -130,7 +130,7 @@ print(fit)
       erase = true
     end
 
-    FileUtils.mkdir_p directory unless File.exists? directory
+    FileUtils.mkdir_p directory unless File.exist? directory
     input_directory  = File.join(directory, 'inputs')
     parameter_chains = File.join(directory, 'chains') unless erase
     summary          = File.join(directory, 'summary') unless erase

data/lib/rbbt/statistics/hypergeometric.rb

@@ -118,7 +118,7 @@ module TSV
 
     field_pos = fields.collect{|f| self.fields.index f}.compact
     persistence_path = self.respond_to?(:persistence_path)? self.persistence_path : nil
-    Persist.persist(filename, :yaml, :fields => fields, :persist => persistence, :prefix => "Hyp.Geo.Counts", :other => {:background => background, :rename => rename, :persistence_path => persistence_path}) do 
+    Persist.persist(filename, :yaml, :fields => fields, :persist => persistence, :prefix => "Hyp.Geo.Counts", :other => {:fields => fields, :background => background, :rename => rename, :persistence_path => persistence_path}) do 
       data ||= {}
 
       with_unnamed do
@@ -152,6 +152,7 @@ module TSV
 
       end
 
+
       if rename
         Log.debug("Using renames during annotation counts")
         Hash[*data.keys.zip(data.values.collect{|l| l.collect{|e| rename.include?(e)? rename[e] : e }.uniq.length }).flatten]

data/lib/rbbt/vector/model/huggingface/masked_lm.rb

@@ -0,0 +1,50 @@
+require 'rbbt/vector/model/huggingface'
+class MaskedLMModel < HuggingfaceModel
+
+  def initialize(checkpoint, dir = nil, model_options = {})
+    
+    model_options = Misc.add_defaults model_options, :max_length => 128
+    super("MaskedLM", checkpoint, dir, model_options)
+
+    train_model do |texts,labels|
+      model, tokenizer = self.init
+      max_length = @model_options[:max_length]
+      mask_id = tokenizer.mask_token_id
+
+      dataset = []
+      texts.zip(labels).each do |text,label_values|
+        fixed_text = text.gsub("[MASK]", "[PENDINGMASK]")
+        label_tokens = label_values.collect{|label| tokenizer.convert_tokens_to_ids(label) }
+        label_tokens.each do |ids|
+          ids = [ids] unless Array === ids
+          fixed_text.sub!("[PENDINGMASK]", "[MASK]" * ids.length)
+        end
+
+        tokenized_text = tokenizer.call(fixed_text, truncation: true, padding: "max_length")
+        input_ids = tokenized_text["input_ids"].to_a
+        attention_mask = tokenized_text["attention_mask"].to_a
+
+        all_label_tokens = label_tokens.flatten
+        label_ids = input_ids.collect do |id|
+          if id == mask_id
+            all_label_tokens.shift
+          else
+            -100
+          end
+        end
+        dataset << {input_ids: input_ids, labels: label_ids, attention_mask: attention_mask}
+      end
+
+      dataset_file = File.join(@directory, 'dataset.json')
+      Open.write(dataset_file, dataset.collect{|e| e.to_json} * "\n")
+
+      training_args_obj = RbbtPython.call_method("rbbt_dm.huggingface", :training_args, @model_path, @model_options[:training_args])
+      data_collator = RbbtPython.class_new_obj("transformers", "DefaultDataCollator", {}) 
+      RbbtPython.call_method("rbbt_dm.huggingface", :train_model, model, tokenizer, training_args_obj, dataset_file, @model_options[:class_weights], data_collator: data_collator)
+
+      model.save_pretrained(@model_path) if @model_path
+      tokenizer.save_pretrained(@model_path) if @model_path
+    end
+
+  end
+end

data/lib/rbbt/vector/model/huggingface.rb

@@ -1,50 +1,68 @@
-require 'rbbt/vector/model'
-require 'rbbt/util/python'
+require 'rbbt/vector/model/torch'
 
-RbbtPython.add_path Rbbt.python.find(:lib)
-RbbtPython.init_rbbt
+class HuggingfaceModel < TorchModel
 
-class HuggingfaceModel < VectorModel
-
-  def self.tsv_dataset(tsv_dataset_file, elements, labels = nil)
+  def self.tsv_dataset(tsv_dataset_file, elements, labels = nil, class_labels = nil)
 
     if labels
+      labels = case class_labels
+               when Array
+                 labels.collect{|l| class_labels.index l}
+               when Hash
+                 inverse_class_labels = {}
+                 class_labels.each{|c,l| inverse_class_labels[l] = c }
+                 labels.collect{|l| inverse_class_labels[l]}
+               else
+                 labels
+               end
+
       Open.write(tsv_dataset_file) do |ffile|
         ffile.puts ["label", "text"].flatten * "\t"
         elements.zip(labels).each do |element,label|
+          element = element.gsub("\n", " ")
           ffile.puts [label, element].flatten * "\t"
         end
+        ffile.sync
       end
     else
       Open.write(tsv_dataset_file) do |ffile|
         ffile.puts ["text"].flatten * "\t"
-        elements.each{|element| ffile.puts element }
+        elements.each do |element|
+          element = element.gsub("\n", " ")
+          ffile.puts element 
+        end
+        ffile.sync
       end
     end
 
     tsv_dataset_file
   end
 
-  def initialize(task, checkpoint, *args)
-    options = args.pop if Hash === args.last
-    options = Misc.add_defaults options, :task => task, :checkpoint => checkpoint
-    super(*args)
-    @model_options ||= {}
-    @model_options.merge!(options)
+  def initialize(task, checkpoint, dir = nil, model_options = {})
+    super(dir, model_options)
 
-    eval_model do |directory,texts|
-      checkpoint = directory && File.directory?(directory) ? directory : @model_options[:checkpoint]
+    @model_options = Misc.add_defaults @model_options, :task => task, :checkpoint => checkpoint
 
-      if @model.nil?
-        @model, @tokenizer = RbbtPython.call_method("rbbt_dm.huggingface", :load_model_and_tokenizer, @model_options[:task], checkpoint)
-      end
-      
-      if Array === texts
+    init_model do 
+      checkpoint = @model_path && File.directory?(@model_path) ? @model_path : @model_options[:checkpoint]
+      model = RbbtPython.call_method("rbbt_dm.huggingface", :load_model, 
+                                     @model_options[:task], checkpoint, **(IndiferentHash.setup(model_options[:model_args]) || {}))
+      tokenizer = RbbtPython.call_method("rbbt_dm.huggingface", :load_tokenizer, 
+                                         @model_options[:task], checkpoint, **(IndiferentHash.setup(model_options[:tokenizer_args]) || {}))
+
+      [model, tokenizer]
+    end
+
+    eval_model do |texts,is_list|
+      model, tokenizer = self.init
+
+      if is_list || @model_options[:task] == "MaskedLM"
+        texts = [texts] if ! is_list
 
         if @model_options.include?(:locate_tokens)
           locate_tokens = @model_options[:locate_tokens]
         elsif @model_options[:task] == "MaskedLM"
-          @model_options[:locate_tokens] = locate_tokens = @tokenizer.special_tokens_map["mask_token"] 
+          @model_options[:locate_tokens] = locate_tokens = tokenizer.special_tokens_map["mask_token"] 
         end
 
         if @directory
@@ -61,18 +79,17 @@ class HuggingfaceModel < VectorModel
         training_args_obj = RbbtPython.call_method("rbbt_dm.huggingface", :training_args, checkpoint_dir, @model_options[:training_args])
 
         begin
-          RbbtPython.call_method("rbbt_dm.huggingface", :predict_model, @model, @tokenizer, training_args_obj, dataset_file, locate_tokens)
+          RbbtPython.call_method("rbbt_dm.huggingface", :predict_model, model, tokenizer, training_args_obj, dataset_file, locate_tokens)
         ensure
           Open.rm_rf tmpdir if tmpdir
         end
       else
-        RbbtPython.call_method("rbbt_dm.huggingface", :eval_model, @model, @tokenizer, [texts], locate_tokens)
+        RbbtPython.call_method("rbbt_dm.huggingface", :eval_model, model, tokenizer, [texts], locate_tokens)
       end
     end
 
-    train_model do |directory,texts,labels|
-      checkpoint = directory && File.directory?(directory) ? directory : @model_options[:checkpoint]
-      @model, @tokenizer = RbbtPython.call_method("rbbt_dm.huggingface", :load_model_and_tokenizer, @model_options[:task], checkpoint)
+    train_model do |texts,labels|
+      model, tokenizer = self.init
 
       if @directory
         tsv_file = File.join(@directory, 'dataset.tsv')
@@ -85,17 +102,19 @@ class HuggingfaceModel < VectorModel
       end
 
       training_args_obj = RbbtPython.call_method("rbbt_dm.huggingface", :training_args, checkpoint_dir, @model_options[:training_args])
-      dataset_file = HuggingfaceModel.tsv_dataset(tsv_file, texts, labels)
+      dataset_file = HuggingfaceModel.tsv_dataset(tsv_file, texts, labels, @model_options[:class_labels])
 
-      RbbtPython.call_method("rbbt_dm.huggingface", :train_model, @model, @tokenizer, training_args_obj, dataset_file, @model_options[:class_weights])
+      RbbtPython.call_method("rbbt_dm.huggingface", :train_model, model, tokenizer, training_args_obj, dataset_file, @model_options[:class_weights])
 
       Open.rm_rf tmpdir if tmpdir
 
-      @model.save_pretrained(directory) if directory
-      @tokenizer.save_pretrained(directory) if directory
+      model.save_pretrained(@model_path) if @model_path
+      tokenizer.save_pretrained(@model_path) if @model_path
     end
 
-    post_process do |result|
+    post_process do |result,is_list|
+      model, tokenizer = self.init
+
       if result.respond_to?(:predictions)
         single = false
         predictions = result.predictions
@@ -137,25 +156,25 @@ class HuggingfaceModel < VectorModel
                        best_token
                      end
 
-                     best.collect{|b| @tokenizer.decode(b) } * "|"
+                     best.collect{|b| tokenizer.decode(b) } * "|"
                    end
                    Array === locate_tokens ? item_masks : item_masks.first
                  end
                else
-                 logits
+                 predictions
                end
 
-      single ? result.first : result
+      (! is_list || single) && Array === result ? result.first : result
     end
 
 
-    save_models if @directory
+    save_models if @model_path
   end
 
   def reset_model
     @model, @tokenizer = nil
-    Open.rm @model_file
+    Open.rm_rf @model_path
+    init
   end
-
 end
 

data/lib/rbbt/vector/model/pytorch_lightning.rb

@@ -0,0 +1,35 @@
+require 'rbbt/vector/model/torch'
+
+class PytorchLightningModel < TorchModel
+  attr_accessor :loader, :val_loader, :trainer
+  def initialize(module_name, class_name, dir = nil, model_options = {})
+    super(dir, model_options)
+    @module_name = module_name
+    @class_name = class_name
+
+    init_model do 
+      RbbtPython.pyimport @module_name
+      RbbtPython.class_new_obj(@module_name, @class_name, @model_options[:model_args] || {})
+    end
+
+    train_model do |features,labels|
+      model = init
+      raise "Use the loader" if @loader.nil?
+      raise "Use the trainer" if @trainer.nil?
+      
+      trainer.fit(model, @loader, @val_loader)
+    end
+
+    eval_model do |features,list|
+      if list
+        model.call(RbbtPython.call_method(:torch, :tensor, features))
+      else
+        model.call(RbbtPython.call_method(:torch, :tensor, [features]))
+      end
+    end
+
+  end
+end
+
+if __FILE__ == $0
+end

data/lib/rbbt/vector/model/random_forest.rb

@@ -27,7 +27,7 @@ label = predict(model, features);
   def importance
     TmpFile.with_file do |tmp|
       tsv = R.run <<-EOF
-load(file="#{model_file}");
+load(file="#{@model_path}");
 rbbt.tsv.write('#{tmp}', model$importance)
       EOF
       TSV.open(tmp)

data/lib/rbbt/vector/model/spaCy.rb

@@ -28,12 +28,12 @@ class SpaCyModel < VectorModel
 
     super(dir)
 
-    @train_model = Proc.new do |file, features, labels|
+    @train_model = Proc.new do |features, labels|
       texts = features
       docs = []
       unique_labels = labels.uniq
-      tmpconfig = File.join(file, 'config')
-      tmptrain = File.join(file, 'train.spacy')
+      tmpconfig = File.join(@model_path, 'config')
+      tmptrain = File.join(@model_path, 'train.spacy')
       SpaCy.config(@config, tmpconfig)
 
       bar = bar(features.length, "Training documents into spacy format")
@@ -54,20 +54,22 @@ class SpaCyModel < VectorModel
       end
 
       gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train, :default => 0)
-      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{file} --paths.train #{tmptrain} --paths.dev #{tmptrain}",  "--gpu-id" => gpu)
+      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{@model_path} --paths.train #{tmptrain} --paths.dev #{tmptrain}",  "--gpu-id" => gpu)
     end
  
-    @eval_model = Proc.new do |file, features,list|
+    @eval_model = Proc.new do |features,list|
       texts = features
       texts = [texts] unless list
 
+      model_path = @model_path
+
       docs = []
       bar = bar(features.length, "Evaluating model")
       SpaCyModel.spacy do
         gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train, :default => 0)
         gpu = gpu.to_i if gpu && gpu != ""
         spacy.require_gpu(gpu) if gpu
-        nlp = spacy.load("#{file}/model-best")
+        nlp = spacy.load("#{model_path}/model-best")
 
         docs = nlp.pipe(texts)
         RbbtPython.collect docs, :bar => bar do |d|
@@ -75,14 +77,6 @@ class SpaCyModel < VectorModel
             d.cats.sort_by{|l,v| v.to_f || 0 }.last.first
           end
         end
-        #nlp.(docs).cats.collect{|cats| cats.sort_by{|l,v| v.to_f }.last.first }
-        #Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
-        #  texts.collect do |text|
-        #    cats = nlp.(text).cats
-        #    bar.tick
-        #    cats.sort_by{|l,v| v.to_f }.last.first
-        #  end
-        #end
       end
     end
   end

data/lib/rbbt/vector/model/tensorflow.rb

@@ -25,7 +25,7 @@ class TensorFlowModel < VectorModel
 
     super(dir)
 
-    @train_model = Proc.new do |file, features, labels|
+    @train_model = Proc.new do |features, labels|
       tensorflow do 
         features = tensorflow.convert_to_tensor(features)
         labels = tensorflow.convert_to_tensor(labels)
@@ -33,16 +33,17 @@ class TensorFlowModel < VectorModel
       @graph ||= keras_graph
       @graph.compile(**@compile_options)
       @graph.fit(features, labels, :epochs => @epochs, :verbose => true)
-      @graph.save(file)
+      @graph.save(@model_path)
     end
  
-    @eval_model = Proc.new do |file, features|
+    @eval_model = Proc.new do |features|
       tensorflow do 
         features = tensorflow.convert_to_tensor(features)
       end
+      model_path = @model_path
+      graph = @graph ||= keras.models.load_model(model_path)
       keras do
-        @graph ||= keras.models.load_model(file)
-        indices = @graph.predict(features, :verbose => false).tolist()
+        indices = graph.predict(features, :verbose => false).tolist()
         labels = indices.collect{|p| p.length > 1 ? p.index(p.max): p.first }
         labels
       end

data/lib/rbbt/vector/model/torch.rb

@@ -0,0 +1,37 @@
+require 'rbbt/vector/model'
+require 'rbbt/util/python'
+
+RbbtPython.add_path Rbbt.python.find(:lib)
+RbbtPython.init_rbbt
+
+class TorchModel < VectorModel
+
+  attr_accessor :model
+
+  def self.get_layer(model, layer)
+    layer.split(".").inject(model){|acc,l| PyCall.getattr(acc, l.to_sym) }
+  end
+
+  def self.get_weights(model, layer)
+    PyCall.getattr(get_layer(model, layer), :weight)
+  end
+
+  def self.freeze(layer)
+    begin
+      PyCall.getattr(layer, :weight).requires_grad = false
+    rescue
+    end
+    RbbtPython.iterate(layer.children) do |layer|
+      freeze(layer)
+    end
+  end
+
+  def self.freeze_layer(model, layer)
+    layer = get_layer(model, layer)
+    freeze(layer)
+  end
+
+  def initialize(dir, model_options = {})
+    super(dir, model_options)
+  end
+end

data/lib/rbbt/vector/model/util.rb

@@ -9,4 +9,22 @@ class VectorModel
     @bar.init
     @bar
   end
+
+  def balance_labels
+    counts = Misc.counts(@labels)
+    min = counts.values.min
+
+    used = {}
+    new_labels = []
+    new_features = []
+    @labels.zip(@features).shuffle.each do |label, features|
+      used[label] ||= 0
+      next if used[label] > min
+      used[label] += 1
+      new_labels << label
+      new_features << features
+    end
+    @labels = new_labels
+    @features = new_features
+  end
 end