rbbt-dm 1.1.48 → 1.1.53

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d054c98df9e1b451849ddcf01f089d2822746d2a4ac8017f7f40f412de5d8c4
-  data.tar.gz: 55df50b751243cf77c023a016b53c258138c24c9b10bf1068fa5bb6385c1c114
+  metadata.gz: b00fd271a576dd8e92f3e24e863ce59128c39edba34c14c75b3f0414f45e2ccf
+  data.tar.gz: 19a2825592b122ab485abaffa432cf19a71afc6f9fc30d6e7a63793fc70de914
 SHA512:
-  metadata.gz: 3b6397645a532feb4bd2aa0dabd715091a2438a07244c11f82ab52f3c40b3fb43e515667b48630e9b8bd28b4d33c0a69cf57d1151dbb0616d1896a4558828fa3
-  data.tar.gz: 20b90f5d900ab8e1a6a930edc5573a420213360961fbb796022e2a09f442307b731c30a7da6b535d7fc90d72fe92fb358de5fdebc420814a7ef300f23c798341
+  metadata.gz: b7b327a6de2ea159266ae41f38a8139e84552ece97b94bdab809dd26474be59b4a62456628257e788384e2cd2d5ea3c6d4f915dfbca1fb24fcad283d75c539ad
+  data.tar.gz: 53581538c5d4ac0a9ff7acda9039565f608caa04e5bfa5721f1e9efc29fb69ff7e1ff88de0379c8ca9be217feb4c273d44fefc5d30f901dcd6c723e0db28abab

data/lib/rbbt/network/paths.rb

@@ -32,7 +32,7 @@ module Paths
     if end_node
       end_node = end_node.select{|n| parents.keys.include? n}.first unless String === end_node
       return nil if not parents.include? end_node
-      extract_path(parents, start_node, u)
+      extract_path(parents, start_node, end_node)
     else
       parents
     end

data/lib/rbbt/tensorflow.rb

@@ -0,0 +1,43 @@
+require 'rbbt/util/python'
+
+module RbbtTensorflow
+
+  def self.init
+    RbbtPython.run do
+      pyimport "tensorflow", as: "tf"
+    end
+  end
+
+  def self.test
+
+    mod = x_test = y_test = nil
+    RbbtPython.run do
+
+      mnist_db = tf.keras.datasets.mnist
+
+      (x_train, y_train), (x_test, y_test) = mnist_db.load_data()
+      x_train, x_test = x_train / 255.0, x_test / 255.0
+
+      mod = tf.keras.models.Sequential.new([
+        tf.keras.layers.Flatten.new(input_shape: [28, 28]),
+        tf.keras.layers.Dense.new(128, activation:'relu'),
+        tf.keras.layers.Dropout.new(0.2),
+        tf.keras.layers.Dense.new(10, activation:'softmax')
+      ])
+      mod.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+      mod.fit(x_train, y_train, epochs:1)
+      mod
+    end
+
+    RbbtPython.run do
+      mod.evaluate(x_test,  y_test, verbose:2)
+    end
+  end
+end
+
+if __FILE__ == $0
+  RbbtTensorflow.init
+  RbbtTensorflow.test
+end

data/lib/rbbt/vector/model.rb

@@ -46,10 +46,10 @@ save(model, file='#{model_file}')
 features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
 load(file="#{model_file}");
 #{code}
-cat(paste(label, sep="\\n"));
+cat(paste(label, sep="\\n", collapse="\\n"));
         EOF
-
-        res = io.read.sub(/WARNING: .*?\n/s,'').split(/\s+/).collect{|l| l.to_f}
+        txt = io.read
+        res = txt.sub(/WARNING: .*?\n/s,'').split(/\s+/).collect{|l| l.to_f}
 
         if list
           res
@@ -60,13 +60,51 @@ cat(paste(label, sep="\\n"));
     end
   end
 
+  def __load_method(file)
+    code = Open.read(file)
+    code.sub!(/.*Proc\.new/, "Proc.new")
+    instance_eval code, file
+  end
+
   def initialize(directory, extract_features = nil, train_model = nil, eval_model = nil)
     @directory = directory
     FileUtils.mkdir_p @directory unless File.exists? @directory
+
     @model_file = File.join(@directory, "model")
-    extract_features = @extract_features 
-    train_model = @train_model 
-    eval_model = @eval_model
+    @extract_features_file = File.join(@directory, "features")
+    @train_model_file = File.join(@directory, "train_model")
+    @eval_model_file = File.join(@directory, "eval_model")
+    @train_model_file_R = File.join(@directory, "train_model.R")
+    @eval_model_file_R = File.join(@directory, "eval_model.R")
+
+    if extract_features.nil?
+      if File.exists?(@extract_features_file)
+        @extract_features = __load_method @extract_features_file
+      end
+    else
+      @extract_features = extract_features 
+    end
+
+    if train_model.nil?
+      if File.exists?(@train_model_file)
+        @train_model = __load_method @train_model_file
+      elsif File.exists?(@train_model_file_R)
+        @train_model = Open.read(@train_model_file_R)
+      end
+    else
+      @train_model = train_model 
+    end
+
+    if eval_model.nil?
+      if File.exists?(@eval_model_file)
+        @eval_model = __load_method @eval_model_file
+      elsif File.exists?(@eval_model_file_R)
+        @eval_model = Open.read(@eval_model_file_R)
+      end
+    else
+      @eval_model = eval_model
+    end
+
     @features = []
     @labels = []
   end
@@ -77,8 +115,47 @@ cat(paste(label, sep="\\n"));
   end
 
   def add(element, label = nil)
-    @features << extract_features.call(element)
-    @labels << label unless label.nil?
+    features = @extract_features ? extract_features.call(element) : element
+    @features << features
+    @labels << label 
+  end
+
+  def add_list(elements, labels = nil)
+    if @extract_features.nil? || @extract_features.arity == 1
+      elements.zip(labels || [nil]).each do |elem,label|
+        add(elem, label)
+      end
+    else
+      features = @extract_features.call(nil, elements)
+      @features.concat  features
+      @labels.concat labels if labels
+    end
+  end
+
+  def save_models
+    require 'method_source'
+
+    case 
+    when Proc === train_model
+      begin
+        Open.write(@train_model_file, train_model.source)
+      rescue
+      end
+    when String === train_model
+      Open.write(@train_model_file_R, @train_model)
+    end
+
+    Open.write(@extract_features_file, @extract_features.source) if @extract_features
+
+    case 
+    when Proc === eval_model
+      begin
+        Open.write(@eval_model_file, eval_model.source)
+      rescue
+      end
+    when String === eval_model
+      Open.write(@eval_model_file_R, eval_model)
+    end
   end
 
   def train
@@ -88,6 +165,7 @@ cat(paste(label, sep="\\n"));
     when String === train_model
       VectorModel.R_train(@model_file,  @features, @labels, train_model)
     end
+    save_models
   end
 
   def run(code)
@@ -96,99 +174,120 @@ cat(paste(label, sep="\\n"));
 
   def eval(element)
     case 
-    when Proc === eval_model
-      eval_model.call(@model_file, extract_features.call(element), false)
-    when String === eval_model
-      VectorModel.R_eval(@model_file,  extract_features.call(element), false, eval_model)
+    when Proc === @eval_model
+      @eval_model.call(@model_file, @extract_features.call(element), false)
+    when String === @eval_model
+      VectorModel.R_eval(@model_file,  @extract_features.call(element), false, eval_model)
     end
   end
 
   def eval_list(elements, extract = true)
+
+    if extract && ! @extract_features.nil? 
+      features = if @extract_features.arity == 1
+                   elements.collect{|element| @extract_features.call(element) }
+                 else
+                   @extract_features.call(nil, elements)
+                 end
+    else
+      features = elements
+    end
+
     case 
     when Proc === eval_model
-      eval_model.call(@model_file, extract ? elements.collect{|element| extract_features.call(element)} : elements, true)
+      eval_model.call(@model_file, features, true)
     when String === eval_model
-      SVMModel.R_eval(@model_file, extract ? elements.collect{|element| extract_features.call(element)} : elements, true, eval_model)
+      VectorModel.R_eval(@model_file, features, true, eval_model)
     end
   end
 
-  def cross_validation(folds = 10)
-    saved_features = @features
-    saved_labels = @labels
-    seq = (0..features.length - 1).to_a
+  #def cross_validation(folds = 10)
+  #  saved_features = @features
+  #  saved_labels = @labels
+  #  seq = (0..features.length - 1).to_a
 
-    chunk_size = features.length / folds
+  #  chunk_size = features.length / folds
 
-    acc = []
-    folds.times do
-      seq = seq.shuffle
-      eval_chunk = seq[0..chunk_size]
-      train_chunk = seq[chunk_size.. -1]
+  #  acc = []
+  #  folds.times do
+  #    seq = seq.shuffle
+  #    eval_chunk = seq[0..chunk_size]
+  #    train_chunk = seq[chunk_size.. -1]
 
-      eval_features = @features.values_at *eval_chunk
-      eval_labels = @labels.values_at *eval_chunk
+  #    eval_features = @features.values_at *eval_chunk
+  #    eval_labels = @labels.values_at *eval_chunk
 
-      @features = @features.values_at *train_chunk
-      @labels = @labels.values_at *train_chunk
+  #    @features = @features.values_at *train_chunk
+  #    @labels = @labels.values_at *train_chunk
 
-      train
-      predictions = eval_list eval_features, false
+  #    train
+  #    predictions = eval_list eval_features, false
 
-      acc << predictions.zip(eval_labels).collect{|pred,lab| pred - lab < 0.5 ? 1 : 0}.inject(0){|acc,e| acc +=e} / chunk_size
+  #    acc << predictions.zip(eval_labels).collect{|pred,lab| pred - lab < 0.5 ? 1 : 0}.inject(0){|acc,e| acc +=e} / chunk_size
 
-      @features = saved_features
-      @labels = saved_labels
-    end
+  #    @features = saved_features
+  #    @labels = saved_labels
+  #  end
 
-    acc
-  end
+  #  acc
+  #end
 
   def cross_validation(folds = 10)
 
     res = TSV.setup({}, "Fold~TP,TN,FP,FN,P,R,F1#:type=:list")
 
-    feature_folds = Misc.divide(@features, folds)
-    labels_folds = Misc.divide(@labels, folds)
+    orig_features = @features
+    orig_labels = @labels
 
-    folds.times do |fix|
+    begin
+      feature_folds = Misc.divide(@features, folds)
+      labels_folds = Misc.divide(@labels, folds)
 
-      test_set = feature_folds[fix]
-      train_set = feature_folds.values_at(*((0..9).to_a - [fix])).inject([]){|acc,e| acc += e; acc}
+      folds.times do |fix|
 
-      test_labels = labels_folds[fix]
-      train_labels = labels_folds.values_at(*((0..9).to_a - [fix])).flatten
+        rest = (0..(folds-1)).to_a - [fix]
 
-      tp, fp, tn, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
+        test_set = feature_folds[fix]
+        train_set = feature_folds.values_at(*rest).inject([]){|acc,e| acc += e; acc}
 
-      @features = train_set
-      @labels = train_labels
-      self.train
-      predictions = self.eval_list test_set, false
+        test_labels = labels_folds[fix]
+        train_labels = labels_folds.values_at(*rest).flatten
 
-      test_labels.zip(predictions).each do |gs,pred|
-        gs = gs.to_i
-        pred = pred > 0.5 ? 1 : 0
-        tp += 1 if gs == pred && gs == 1
-        tn += 1 if gs == pred && gs == 0
-        fp += 1 if gs == 0 && pred == 1
-        fn += 1 if gs == 1 && pred == 0
-      end
+        tp, fp, tn, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
 
-      p = tp + fn
-      pp = tp + fp
+        @features = train_set
+        @labels = train_labels
+        self.train
+        predictions = self.eval_list test_set, false
 
-      pr = tp.to_f / pp
-      re = tp.to_f / p
+        raise "Number of predictions (#{predictions.length}) and test labels (#{test_labels.length}) do not match" if predictions.length != test_labels.length
 
-      f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
+        test_labels.zip(predictions).each do |gs,pred|
+          gs = gs.to_i
+          pred = pred > 0.5 ? 1 : 0
+          tp += 1 if gs == pred && gs == 1
+          tn += 1 if gs == pred && gs == 0
+          fp += 1 if gs == 0 && pred == 1
+          fn += 1 if gs == 1 && pred == 0
+        end
 
-      Misc.fingerprint([tp,tn,fp,fn,pr,re,f1])
+        p = tp + fn
+        pp = tp + fp
 
-      Log.debug "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1}"
+        pr = tp.to_f / pp
+        re = tp.to_f / p
 
-      res[fix] = [tp,tn,fp,fn,pr,re,f1]
-    end
+        f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
+
+        Log.debug "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1} - #{[tp.to_s, tn.to_s, fp.to_s, fn.to_s] * " "}"
 
+        res[fix] = [tp,tn,fp,fn,pr,re,f1]
+      end
+    ensure
+      @features = orig_features
+      @labels = orig_labels
+    end
+    self.train
     res
   end
 end

data/lib/rbbt/vector/model/spaCy.rb

@@ -0,0 +1,76 @@
+require 'rbbt/vector/model'
+require 'rbbt/nlp/spaCy'
+
+class SpaCyModel < VectorModel
+  attr_accessor :config
+
+  def spacy(&block)
+    RbbtPython.run "spacy" do 
+      RbbtPython.module_eval(&block)
+    end
+  end
+  
+  def initialize(dir, config, lang = 'en_core_web_md')
+    @config = case
+              when Path === config
+                config.read
+              when Misc.is_filename?(config)
+                Open.read(config)
+              when (Misc.is_filename?(config, false) && Rbbt.share.spaCy.cpu[config].exists?)
+                Rbbt.share.spaCy.cpu[config].read
+              when (Misc.is_filename?(config, false) && Rbbt.share.spaCy[config].exists?)
+                Rbbt.share.spaCy[config].read
+              else
+                config
+              end
+    @lang = lang
+
+    super(dir)
+
+    @train_model = Proc.new do |file, features, labels|
+      texts = features
+      docs = []
+      tmpconfig = File.join(file, 'config')
+      tmptrain = File.join(file, 'train.spacy')
+      SpaCy.config(@config, tmpconfig)
+      spacy do
+        nlp = SpaCy.nlp(lang)
+        docs = []
+        RbbtPython.iterate nlp.pipe(texts.zip(labels), as_tuples: true), :bar => "Training documents into spacy format" do |doc,label|
+          if %w(1 true pos).include?(label.to_s.downcase) 
+            doc.cats["positive"] = 1
+            doc.cats["negative"] = 0
+          else
+            doc.cats["positive"] = 0
+            doc.cats["negative"] = 1
+          end
+          docs << doc
+        end
+
+        doc_bin = spacy.tokens.DocBin.new(docs: docs)
+        doc_bin.to_disk(tmptrain)
+      end
+
+      gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train)
+      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{file} --paths.train #{tmptrain} --paths.dev #{tmptrain}",  "--gpu-id" => gpu)
+    end
+ 
+    @eval_model = Proc.new do |file, features|
+      texts = features
+
+      docs = []
+      spacy do
+        nlp = spacy.load("#{file}/model-best")
+
+        Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
+          texts.collect do |text|
+            cats = nlp.(text).cats
+            bar.tick
+            cats['positive'] > cats['negative']  ? 1 : 0
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/vector/model/svm.rb

@@ -9,7 +9,7 @@ class SVMModel < VectorModel
 
     @train_model =<<-EOF
 library(e1071);
-model = svm(class ~ ., data = features, scale=c(0));
+model = svm(as.factor(class) ~ ., data = features);
     EOF
  
     @eval_model =<<-EOF