rbbt-dm 1.1.46 → 1.1.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f70d6a55c5852ffdafd59a3199b41031ee5a5aaa4becc56ee2f7f49be3b5d43
-  data.tar.gz: f5d53ca95b6af4d6c2b63f00bbbfd398e3fa5a6feea15885b00ccd946f3aa765
+  metadata.gz: 5327d1c2a46283b95fa380a73be418cf7e5a3afd2aca6002bd9ca591ab3f6df8
+  data.tar.gz: 1066345322e342c8f642b89825c1a8555c6bfa1d09985705d1b382654d91653f
 SHA512:
-  metadata.gz: 7cd9928cd7a3e7558e27796f0e8f4c7635caa2331dc3f2d2d904434d9feedf6e3d2085ba301b55c61b87bdee83fbc588c666a99c0d0cc0d5414d1614a964a0a5
-  data.tar.gz: f559bff1acc000be594ad8e91534a4af24acad3ebf80c276b8055af582a91e5137c748acf55267c8001b0c667cc1c6a167ca86a11b34da52ebb796ba8fdd01fe
+  metadata.gz: c1a3cf2ec93909993b290c7c6cb0b6e9c6090155657403c705b93b74a538cbe91ff23dead14c33453dde0c31ba681099b3e5c93f2699a471c19c299b43d0f304
+  data.tar.gz: e5b456330625bb57a494fb9e5fc9757e96c134da8f410fabe8f7e9d06169d09f0c4fa3c7e5a375870e6f45b8f5ffdf7855b8260719fb1a32846a2f24c18e8853

data/lib/rbbt/matrix/barcode.rb

@@ -38,7 +38,7 @@ rbbt.GE.barcode.mode(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{ R.r
           end
         end
         key = key.first if Array === key
-      [key, bars]
+        [key, bars]
       end
     end
 

data/lib/rbbt/network/paths.rb

@@ -32,7 +32,7 @@ module Paths
     if end_node
       end_node = end_node.select{|n| parents.keys.include? n}.first unless String === end_node
       return nil if not parents.include? end_node
-      extract_path(parents, start_node, u)
+      extract_path(parents, start_node, end_node)
     else
       parents
     end

data/lib/rbbt/tensorflow.rb

@@ -0,0 +1,43 @@
+require 'rbbt/util/python'
+
+module RbbtTensorflow
+
+  def self.init
+    RbbtPython.run do
+      pyimport "tensorflow", as: "tf"
+    end
+  end
+
+  def self.test
+
+    mod = x_test = y_test = nil
+    RbbtPython.run do
+
+      mnist_db = tf.keras.datasets.mnist
+
+      (x_train, y_train), (x_test, y_test) = mnist_db.load_data()
+      x_train, x_test = x_train / 255.0, x_test / 255.0
+
+      mod = tf.keras.models.Sequential.new([
+        tf.keras.layers.Flatten.new(input_shape: [28, 28]),
+        tf.keras.layers.Dense.new(128, activation:'relu'),
+        tf.keras.layers.Dropout.new(0.2),
+        tf.keras.layers.Dense.new(10, activation:'softmax')
+      ])
+      mod.compile(optimizer='adam',
+                  loss='sparse_categorical_crossentropy',
+                  metrics=['accuracy'])
+      mod.fit(x_train, y_train, epochs:1)
+      mod
+    end
+
+    RbbtPython.run do
+      mod.evaluate(x_test,  y_test, verbose:2)
+    end
+  end
+end
+
+if __FILE__ == $0
+  RbbtTensorflow.init
+  RbbtTensorflow.test
+end

data/lib/rbbt/vector/model.rb

@@ -4,6 +4,20 @@ class VectorModel
   attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model
   attr_accessor :features, :labels
 
+  def self.R_run(model_file, features, labels, code)
+    TmpFile.with_file do |feature_file|
+      Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+      Open.write(feature_file + '.class', labels * "\n")
+
+      R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class");
+features = cbind(features, class = labels);
+#{code}
+      EOF
+    end
+  end
+
   def self.R_train(model_file, features, labels, code)
     TmpFile.with_file do |feature_file|
       Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
@@ -32,10 +46,10 @@ save(model, file='#{model_file}')
 features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
 load(file="#{model_file}");
 #{code}
-cat(paste(label, sep="\\n"));
+cat(paste(label, sep="\\n", collapse="\\n"));
         EOF
-
-        res = io.read.sub(/WARNING: .*?\n/s,'').split(/\s+/).collect{|l| l.to_f}
+        txt = io.read
+        res = txt.sub(/WARNING: .*?\n/s,'').split(/\s+/).collect{|l| l.to_f}
 
         if list
           res
@@ -46,13 +60,51 @@ cat(paste(label, sep="\\n"));
     end
   end
 
+  def __load_method(file)
+    code = Open.read(file)
+    code.sub!(/.*Proc\.new/, "Proc.new")
+    instance_eval code, file
+  end
+
   def initialize(directory, extract_features = nil, train_model = nil, eval_model = nil)
     @directory = directory
     FileUtils.mkdir_p @directory unless File.exists? @directory
+
     @model_file = File.join(@directory, "model")
-    extract_features = @extract_features 
-    train_model = @train_model 
-    eval_model = @eval_model
+    @extract_features_file = File.join(@directory, "features")
+    @train_model_file = File.join(@directory, "train_model")
+    @eval_model_file = File.join(@directory, "eval_model")
+    @train_model_file_R = File.join(@directory, "train_model.R")
+    @eval_model_file_R = File.join(@directory, "eval_model.R")
+
+    if extract_features.nil?
+      if File.exists?(@extract_features_file)
+        @extract_features = __load_method @extract_features_file
+      end
+    else
+      @extract_features = extract_features 
+    end
+
+    if train_model.nil?
+      if File.exists?(@train_model_file)
+        @train_model = __load_method @train_model_file
+      elsif File.exists?(@train_model_file_R)
+        @train_model = Open.read(@train_model_file_R)
+      end
+    else
+      @train_model = train_model 
+    end
+
+    if eval_model.nil?
+      if File.exists?(@eval_model_file)
+        @eval_model = __load_method @eval_model_file
+      elsif File.exists?(@eval_model_file_R)
+        @eval_model = Open.read(@eval_model_file_R)
+      end
+    else
+      @eval_model = eval_model
+    end
+
     @features = []
     @labels = []
   end
@@ -63,8 +115,47 @@ cat(paste(label, sep="\\n"));
   end
 
   def add(element, label = nil)
-    @features << extract_features.call(element)
-    @labels << label unless label.nil?
+    features = @extract_features ? extract_features.call(element) : element
+    @features << features
+    @labels << label 
+  end
+
+  def add_list(elements, labels = nil)
+    if @extract_features.nil? || @extract_features.arity == 1
+      elements.zip(labels || [nil]).each do |elem,label|
+        add(elem, label)
+      end
+    else
+      features = @extract_features.call(nil, elements)
+      @features.concat  features
+      @labels.concat labels if labels
+    end
+  end
+
+  def save_models
+    require 'method_source'
+
+    case 
+    when Proc === train_model
+      begin
+        Open.write(@train_model_file, train_model.source)
+      rescue
+      end
+    when String === train_model
+      Open.write(@train_model_file_R, @train_model)
+    end
+
+    Open.write(@extract_features_file, @extract_features.source) if @extract_features
+
+    case 
+    when Proc === eval_model
+      begin
+        Open.write(@eval_model_file, eval_model.source)
+      rescue
+      end
+    when String === eval_model
+      Open.write(@eval_model_file_R, eval_model)
+    end
   end
 
   def train
@@ -72,105 +163,131 @@ cat(paste(label, sep="\\n"));
     when Proc === train_model
       train_model.call(@model_file, @features, @labels)
     when String === train_model
-      SVMModel.R_train(@model_file,  @features, @labels, train_model)
+      VectorModel.R_train(@model_file,  @features, @labels, train_model)
     end
+    save_models
+  end
+
+  def run(code)
+    VectorModel.R_run(@model_file,  @features, @labels, code)
   end
 
   def eval(element)
     case 
-    when Proc === eval_model
-      eval_model.call(@model_file, extract_features.call(element), false)
-    when String === eval_model
-      SVMModel.R_eval(@model_file,  extract_features.call(element), false, eval_model)
+    when Proc === @eval_model
+      @eval_model.call(@model_file, @extract_features.call(element), false)
+    when String === @eval_model
+      VectorModel.R_eval(@model_file,  @extract_features.call(element), false, eval_model)
     end
   end
 
   def eval_list(elements, extract = true)
+
+    if extract && ! @extract_features.nil? 
+      features = if @extract_features.arity == 1
+                   elements.collect{|element| @extract_features.call(element) }
+                 else
+                   @extract_features.call(nil, elements)
+                 end
+    else
+      features = elements
+    end
+
     case 
     when Proc === eval_model
-      eval_model.call(@model_file, extract ? elements.collect{|element| extract_features.call(element)} : elements, true)
+      eval_model.call(@model_file, features, true)
     when String === eval_model
-      SVMModel.R_eval(@model_file, extract ? elements.collect{|element| extract_features.call(element)} : elements, true, eval_model)
+      VectorModel.R_eval(@model_file, features, true, eval_model)
     end
   end
 
-  def cross_validation(folds = 10)
-    saved_features = @features
-    saved_labels = @labels
-    seq = (0..features.length - 1).to_a
+  #def cross_validation(folds = 10)
+  #  saved_features = @features
+  #  saved_labels = @labels
+  #  seq = (0..features.length - 1).to_a
 
-    chunk_size = features.length / folds
+  #  chunk_size = features.length / folds
 
-    acc = []
-    folds.times do
-      seq = seq.shuffle
-      eval_chunk = seq[0..chunk_size]
-      train_chunk = seq[chunk_size.. -1]
+  #  acc = []
+  #  folds.times do
+  #    seq = seq.shuffle
+  #    eval_chunk = seq[0..chunk_size]
+  #    train_chunk = seq[chunk_size.. -1]
 
-      eval_features = @features.values_at *eval_chunk
-      eval_labels = @labels.values_at *eval_chunk
+  #    eval_features = @features.values_at *eval_chunk
+  #    eval_labels = @labels.values_at *eval_chunk
 
-      @features = @features.values_at *train_chunk
-      @labels = @labels.values_at *train_chunk
+  #    @features = @features.values_at *train_chunk
+  #    @labels = @labels.values_at *train_chunk
 
-      train
-      predictions = eval_list eval_features, false
+  #    train
+  #    predictions = eval_list eval_features, false
 
-      acc << predictions.zip(eval_labels).collect{|pred,lab| pred - lab < 0.5 ? 1 : 0}.inject(0){|acc,e| acc +=e} / chunk_size
+  #    acc << predictions.zip(eval_labels).collect{|pred,lab| pred - lab < 0.5 ? 1 : 0}.inject(0){|acc,e| acc +=e} / chunk_size
 
-      @features = saved_features
-      @labels = saved_labels
-    end
+  #    @features = saved_features
+  #    @labels = saved_labels
+  #  end
 
-    acc
-  end
+  #  acc
+  #end
 
   def cross_validation(folds = 10)
 
     res = TSV.setup({}, "Fold~TP,TN,FP,FN,P,R,F1#:type=:list")
 
-    feature_folds = Misc.divide(@features, folds)
-    labels_folds = Misc.divide(@labels, folds)
+    orig_features = @features
+    orig_labels = @labels
 
-    folds.times do |fix|
+    begin
+      feature_folds = Misc.divide(@features, folds)
+      labels_folds = Misc.divide(@labels, folds)
 
-      test_set = feature_folds[fix]
-      train_set = feature_folds.values_at(*((0..9).to_a - [fix])).inject([]){|acc,e| acc += e; acc}
+      folds.times do |fix|
 
-      test_labels = labels_folds[fix]
-      train_labels = labels_folds.values_at(*((0..9).to_a - [fix])).flatten
+        rest = (0..(folds-1)).to_a - [fix]
 
-      tp, fp, tn, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
+        test_set = feature_folds[fix]
+        train_set = feature_folds.values_at(*rest).inject([]){|acc,e| acc += e; acc}
 
-      @features = train_set
-      @labels = train_labels
-      self.train
-      predictions = self.eval_list test_set, false
+        test_labels = labels_folds[fix]
+        train_labels = labels_folds.values_at(*rest).flatten
 
-      test_labels.zip(predictions).each do |gs,pred|
-        gs = gs.to_i
-        pred = pred > 0.5 ? 1 : 0
-        tp += 1 if gs == pred && gs == 1
-        tn += 1 if gs == pred && gs == 0
-        fp += 1 if gs == 0 && pred == 1
-        fn += 1 if gs == 1 && pred == 0
-      end
+        tp, fp, tn, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
 
-      p = tp + fn
-      pp = tp + fp
+        @features = train_set
+        @labels = train_labels
+        self.train
+        predictions = self.eval_list test_set, false
 
-      pr = tp.to_f / pp
-      re = tp.to_f / p
+        raise "Number of predictions (#{predictions.length}) and test labels (#{test_labels.length}) do not match" if predictions.length != test_labels.length
 
-      f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
+        test_labels.zip(predictions).each do |gs,pred|
+          gs = gs.to_i
+          pred = pred > 0.5 ? 1 : 0
+          tp += 1 if gs == pred && gs == 1
+          tn += 1 if gs == pred && gs == 0
+          fp += 1 if gs == 0 && pred == 1
+          fn += 1 if gs == 1 && pred == 0
+        end
 
-      Misc.fingerprint([tp,tn,fp,fn,pr,re,f1])
+        p = tp + fn
+        pp = tp + fp
 
-      Log.debug "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1}"
+        pr = tp.to_f / pp
+        re = tp.to_f / p
 
-      res[fix] = [tp,tn,fp,fn,pr,re,f1]
-    end
+        f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
 
+        Log.debug "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1} - #{[tp.to_s, tn.to_s, fp.to_s, fn.to_s] * " "}"
+
+        res[fix] = [tp,tn,fp,fn,pr,re,f1]
+      end
+    ensure
+      @features = orig_features
+      @labels = orig_labels
+    end
+    self.train
     res
   end
 end

data/lib/rbbt/vector/model/spaCy.rb

@@ -0,0 +1,73 @@
+require 'rbbt/vector/model'
+require 'rbbt/nlp/spaCy'
+
+class SpaCyModel < VectorModel
+  attr_accessor :config
+
+  def spacy(&block)
+    RbbtPython.run "spacy" do 
+      RbbtPython.module_eval(&block)
+    end
+  end
+  
+  def initialize(dir, config, lang = 'en_core_web_md')
+    @config = case
+              when Path === config
+                config.read
+              when Misc.is_filename?(config)
+                Open.read(config)
+              when (Misc.is_filename?(config, false) && Rbbt.share.spaCy.cpu[config].exists?)
+                Rbbt.share.spaCy.cpu[config].read
+              when (Misc.is_filename?(config, false) && Rbbt.share.spaCy[config].exists?)
+                Rbbt.share.spaCy[config].read
+              else
+                config
+              end
+    @lang = lang
+
+    super(dir)
+
+    @train_model = Proc.new do |file, features, labels|
+      texts = features
+      docs = []
+      tmpconfig = File.join(file, 'config')
+      tmptrain = File.join(file, 'train.spacy')
+      SpaCy.config(@config, tmpconfig)
+      spacy do
+        nlp = SpaCy.nlp(lang)
+        docs = []
+        RbbtPython.iterate nlp.pipe(texts.zip(labels), as_tuples: true), :bar => "Training documents into spacy format" do |doc,label|
+          if %w(1 true pos).include?(label.to_s.downcase) 
+            doc.cats["positive"] = 1
+            doc.cats["negative"] = 0
+          else
+            doc.cats["positive"] = 0
+            doc.cats["negative"] = 1
+          end
+          docs << doc
+        end
+
+        doc_bin = spacy.tokens.DocBin.new(docs: docs)
+        doc_bin.to_disk(tmptrain)
+      end
+
+      gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train)
+      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{file} --paths.train #{tmptrain} --paths.dev #{tmptrain}",  "--gpu-id" => gpu)
+    end
+ 
+    @eval_model = Proc.new do |file, features|
+      texts = features
+
+      docs = []
+      spacy do
+        nlp = spacy.load("#{file}/model-best")
+
+        texts.collect do |text|
+          cats = nlp.(text).cats
+          cats['positive'] > cats['negative']  ? 1 : 0
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/vector/model/svm.rb

@@ -9,7 +9,7 @@ class SVMModel < VectorModel
 
     @train_model =<<-EOF
 library(e1071);
-model = svm(class ~ ., data = features, scale=c(0));
+model = svm(as.factor(class) ~ ., data = features);
     EOF
  
     @eval_model =<<-EOF

data/lib/rbbt/vector/model/tensorflow.rb

@@ -0,0 +1,55 @@
+require 'rbbt/vector/model'
+require 'rbbt/tensorflow'
+
+class TensorFlowModel < VectorModel
+  attr_accessor :graph, :epochs, :compile_options
+
+  def tensorflow(&block)
+    RbbtPython.run "tensorflow" do 
+      RbbtPython.module_eval(&block)
+    end
+  end
+
+  def keras(&block)
+    RbbtPython.run "tensorflow.keras", as: 'keras' do 
+      RbbtPython.run "tensorflow" do 
+        RbbtPython.module_eval(&block)
+      end
+    end
+  end
+  
+  def initialize(dir, graph = nil, epochs = 3, **compile_options)
+    @graph = graph
+    @epochs = epochs
+    @compile_options = compile_options
+
+    super(dir)
+
+    @train_model = Proc.new do |file, features, labels|
+      tensorflow do 
+        features = tensorflow.convert_to_tensor(features)
+        labels = tensorflow.convert_to_tensor(labels)
+      end
+      @graph ||= keras_graph
+      @graph.compile(**@compile_options)
+      @graph.fit(features, labels, :epochs => @epochs, :verbose => false)
+      @graph.save(file)
+    end
+ 
+    @eval_model = Proc.new do |file, features|
+      tensorflow do 
+        features = tensorflow.convert_to_tensor(features)
+      end
+      keras do
+        @graph ||= keras.models.load_model(file)
+        indices = @graph.predict(features, :verbose => false).tolist()
+        labels = indices.collect{|p| p.length > 1 ? p.index(p.max): p.first }
+        labels
+      end
+    end
+  end
+
+  def keras_graph(&block)
+    @graph = keras(&block)
+  end
+end

data/test/rbbt/matrix/test_barcode.rb

@@ -11,7 +11,7 @@ class TestBarcode < Test::Unit::TestCase
     data["G4"] = [6,6,1,1,1,1]
 
     TmpFile.with_file(data.to_s) do |file|
-      m = Matrix.new file
+      m = RbbtMatrix.new file
       m.barcode(file+'.barcode')
       tsv =  TSV.open(file+'.barcode')
       assert tsv["G2"] = [0,1,0,1,0,1]

data/test/rbbt/network/test_paths.rb

@@ -20,7 +20,7 @@ N4 N5
     end_node = "N5"
 
     path = Paths.dijkstra(network, start_node, [end_node])
-    assert_equal %w(N1 N2 N4), path.reverse
+    assert_equal %w(N1 N2 N4 N5), path.reverse
   end
 
   def test_weighted_dijsktra

data/test/rbbt/vector/model/test_spaCy.rb

@@ -0,0 +1,121 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/vector/model/spaCy'
+require 'rbbt/vector/model/svm'
+
+class TestSpaCyModel < Test::Unit::TestCase
+
+  def test_spyCy
+    TmpFile.with_file() do |dir|
+      Log.severity = 0
+      FileUtils.mkdir_p dir
+
+      model = SpaCyModel.new(
+        dir, 
+        "cpu/textcat_efficiency.conf"
+      )
+
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      good.keys[0..gsize-1].each do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      bad.keys[0..bsize-1].each do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+    end
+
+    def test_svm_spacy
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      model = SVMModel.new(
+        dir
+      )
+
+      nlp = RbbtPython.run "spacy" do
+        spacy.load('en_core_web_md')
+      end
+
+      model.extract_features = Proc.new do |text|
+        vs = RbbtPython.run do
+          RbbtPython.collect nlp.(text).__iter__ do |token|
+            token.vector.tolist()
+          end
+        end
+        length = vs.length
+
+        v = vs.inject(nil){|acc,ev| acc = acc.nil? ? ev : acc.zip(ev).collect{|a,b| a + b } }
+
+        v.collect{|e| e / length }
+      end
+
+      TSV.traverse good.keys[0..gsize-1], :type => :array, :bar => true do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      TSV.traverse bad.keys[0..bsize-1], :type => :array, :bar => true do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+
+    end
+  end
+
+  def test_spyCy_trf
+    TmpFile.with_file() do |dir|
+      Log.severity = 0
+      FileUtils.mkdir_p dir
+
+      model = SpaCyModel.new(
+        dir, 
+        "gpu/textcat_accuracy.conf"
+      )
+
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      good.keys[0..gsize-1].each do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      bad.keys[0..bsize-1].each do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+    end
+  end
+end
+

data/test/rbbt/vector/model/test_tensorflow.rb

@@ -0,0 +1,57 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/vector/model/tensorflow'
+
+class TestTensorflowModel < Test::Unit::TestCase
+
+  def test_keras
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+
+      model = TensorFlowModel.new(
+        dir, 
+        optimizer:'adam',
+        loss: 'sparse_categorical_crossentropy',
+        metrics: ['accuracy']
+      )
+
+      model.keras_graph do
+        tf = tensorflow
+        tf.keras.models.Sequential.new([
+          tf.keras.layers.Flatten.new(input_shape: [28, 28]),
+          tf.keras.layers.Dense.new(128, activation:'relu'),
+          tf.keras.layers.Dropout.new(0.2),
+          tf.keras.layers.Dense.new(10, activation:'softmax')
+        ])
+      end
+
+      sum = predictions = nil
+      model.tensorflow do
+        tf = tensorflow
+        mnist_db = tf.keras.datasets.mnist
+
+        (x_train, y_train), (x_test, y_test) = mnist_db.load_data()
+        x_train, x_test = x_train / 255.0, x_test / 255.0
+
+        num = PyCall.len(x_train)
+
+        num.times do |i|
+          model.add x_train[i], y_train[i]
+        end
+
+        model.train
+
+        predictions = model.eval_list x_test.tolist()
+        sum = 0
+        predictions.zip(y_test.tolist()).each do |pred,label|
+          sum += 1 if label.to_i == pred
+        end
+
+      end
+
+      assert sum.to_f / predictions.length > 0.7
+
+
+    end
+  end
+end
+

data/test/rbbt/vector/test_model.rb

@@ -71,4 +71,358 @@ cat(label, file="#{results}");
     end
   end
 
+  def test_model_list
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;1;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
+        end
+      }
+
+      model.train_model = Proc.new{|model_file,features,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features) 
+save(model, file="#{ model_file }");
+          EOF
+        end
+      }
+
+      model.eval_model = Proc.new{|model_file,features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            puts R.run(<<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ model_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+            ).read
+            Open.read(results)
+          end
+        end
+
+      }
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        [features, label]
+      end
+
+      model.add_list(*Misc.zip_fields(pairs))
+
+      model.train
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+    end
+  end
+
+  def test_model_list2
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;1;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.extract_features = Proc.new{|element|
+        element.split(";")
+      }
+
+      model.train_model = Proc.new{|model_file,features,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features) 
+save(model, file="#{ model_file }");
+          EOF
+        end
+      }
+
+      model.eval_model = Proc.new{|model_file,features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            puts R.run(<<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ model_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+            ).read
+            Open.read(results)
+          end
+        end
+
+      }
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        [features, label]
+      end
+
+      model.add_list(*Misc.zip_fields(pairs))
+
+      model.train
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+    end
+  end
+
+  def test_model_list
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;1;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
+        end
+      }
+
+      model.train_model = Proc.new{|model_file,features,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features) 
+save(model, file="#{ model_file }");
+          EOF
+        end
+      }
+
+      model.eval_model = Proc.new{|model_file,features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            puts R.run(<<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ model_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+            ).read
+            Open.read(results)
+          end
+        end
+
+      }
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        model.add features, label
+      end
+
+      model.train
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+    end
+  end
+
+  def test_model_save
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;1;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.extract_features = Proc.new{|element|
+        element.split(";")
+      }
+
+      model.train_model = Proc.new{|model_file,features,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features) 
+save(model, file="#{ model_file }");
+          EOF
+        end
+      }
+
+      model.eval_model = Proc.new{|model_file,features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            puts R.run(<<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ model_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+            ).read
+            Open.read(results)
+          end
+        end
+
+      }
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        [features, label]
+      end
+
+      model.add_list(*Misc.zip_fields(pairs))
+
+      model.train
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+    end
+  end
+
+  def test_model_save
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;1;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
+        end
+      }
+
+      model.train_model = Proc.new{|model_file,features,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features) 
+save(model, file="#{ model_file }");
+          EOF
+        end
+      }
+
+      model.eval_model = Proc.new{|model_file,features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            puts R.run(<<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ model_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+            ).read
+            Open.read(results)
+          end
+        end
+
+      }
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        model.add features, label
+      end
+
+      model.train
+
+      model = VectorModel.new(dir)
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        model.add features, label
+      end
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.46
+  version: 1.1.51
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-30 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: priority_queue_cxx
+  name: priority_queue_cxx17
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -105,8 +105,11 @@ files:
 - lib/rbbt/statistics/hypergeometric.rb
 - lib/rbbt/statistics/random_walk.rb
 - lib/rbbt/statistics/rank_product.rb
+- lib/rbbt/tensorflow.rb
 - lib/rbbt/vector/model.rb
+- lib/rbbt/vector/model/spaCy.rb
 - lib/rbbt/vector/model/svm.rb
+- lib/rbbt/vector/model/tensorflow.rb
 - share/R/MA.R
 - share/R/barcode.R
 - share/R/heatmap.3.R
@@ -118,7 +121,9 @@ files:
 - test/rbbt/statistics/test_random_walk.rb
 - test/rbbt/test_ml_task.rb
 - test/rbbt/test_stan.rb
+- test/rbbt/vector/model/test_spaCy.rb
 - test/rbbt/vector/model/test_svm.rb
+- test/rbbt/vector/model/test_tensorflow.rb
 - test/rbbt/vector/test_model.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-phgx
@@ -139,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Data-mining and statistics
@@ -152,6 +157,8 @@ test_files:
 - test/rbbt/statistics/test_hypergeometric.rb
 - test/rbbt/test_ml_task.rb
 - test/rbbt/vector/test_model.rb
+- test/rbbt/vector/model/test_spaCy.rb
+- test/rbbt/vector/model/test_tensorflow.rb
 - test/rbbt/vector/model/test_svm.rb
 - test/rbbt/test_stan.rb
 - test/test_helper.rb