rbbt-dm 1.1.50 → 1.1.54

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2efbfe29d382655f78a91a1e8e381a304f4610f2d13d6d7c5fae8be40a22aa29
-  data.tar.gz: cee71dc48458dad863d6cc04bf8d1c113430e3f78309dac4cad1d575d9f4b198
+  metadata.gz: 40ee19cdf9fd742bfa844428a16e61f398f76d53ca0bfdda5499f5bb03db1c2b
+  data.tar.gz: 7be309bf582ce9e547cf316a36b88ad3043fb0bdc9cd5d5590d507f0f78a7a71
 SHA512:
-  metadata.gz: 953cf89ffc4a04e38aff7020bd25d8ab8db5a6383db3a9a9fce5e7bab3bc668a763d660953bdbeedfc9d32beaacc6e83142c773dac6663b118c3a02c58405611
-  data.tar.gz: c899b27449bb378c9c757438aed976d5375e8d49bb99edf80ff7dbf9f2b03adeac9ec2000aa35bed4206b13e399f025e326952e35acfaefe04d1e5612597367f
+  metadata.gz: 801e3c9a8541b3b87b12b961c31c4932c17b36db6c9d65dd9a88ef1adb5811188d79570099cf96a1dea3d71d6323edfe2707fd46ebe9e9faa96f38ade84684eb
+  data.tar.gz: e1955147d51d34595cf169e818da48584cf9f1597dd2a714d91de88cf90846e9845f3147715ea95769cfb453adf170879c926e02419ff1437b95968cf9bf9ffc

data/lib/rbbt/tensorflow.rb

@@ -27,7 +27,7 @@ module RbbtTensorflow
       mod.compile(optimizer='adam',
                   loss='sparse_categorical_crossentropy',
                   metrics=['accuracy'])
-      mod.fit(x_train, y_train, epochs:1)
+      mod.fit(x_train, y_train, epochs:3)
       mod
     end
 

data/lib/rbbt/vector/model/random_forest.rb

@@ -0,0 +1,26 @@
+require 'rbbt/vector/model'
+class RFModel < VectorModel
+  def initialize(dir)
+    super(dir)
+
+    @extract_features = Proc.new{|element|
+      element
+    }
+
+    @train_model =<<-EOF
+rbbt.require("randomForest");
+model = randomForest(as.factor(label) ~ ., data = features);
+    EOF
+ 
+    @eval_model =<<-EOF
+rbbt.require("randomForest");
+pred = names(model$forest$xlevels)
+for (p in pred) { 
+  if (class(features[[p]]) == "factor") { 
+      features[[p]] = factor(features[[p]], levels=model$forest$xlevels[[p]])
+    } 
+}
+label = predict(model, features);
+    EOF
+  end
+end

data/lib/rbbt/vector/model/spaCy.rb

@@ -51,7 +51,8 @@ class SpaCyModel < VectorModel
         doc_bin.to_disk(tmptrain)
       end
 
-      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{file} --paths.train #{tmptrain} --paths.dev #{tmptrain}")
+      gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train)
+      CMD.cmd_log(:spacy, "train #{tmpconfig} --output #{file} --paths.train #{tmptrain} --paths.dev #{tmptrain}",  "--gpu-id" => gpu)
     end
  
     @eval_model = Proc.new do |file, features|
@@ -61,9 +62,12 @@ class SpaCyModel < VectorModel
       spacy do
         nlp = spacy.load("#{file}/model-best")
 
-        texts.collect do |text|
-          cats = nlp.(text).cats
-          cats['positive'] > cats['negative']  ? 1 : 0
+        Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
+          texts.collect do |text|
+            cats = nlp.(text).cats
+            bar.tick
+            cats['positive'] > cats['negative']  ? 1 : 0
+          end
         end
       end
     end

data/lib/rbbt/vector/model/svm.rb

@@ -8,12 +8,12 @@ class SVMModel < VectorModel
     }
 
     @train_model =<<-EOF
-library(e1071);
-model = svm(as.factor(class) ~ ., data = features);
+rbbt.require('e1071');
+model = svm(as.factor(label) ~ ., data = features);
     EOF
  
     @eval_model =<<-EOF
-library(e1071);
+rbbt.require('e1071');
 label = predict(model, features);
     EOF
   end

data/lib/rbbt/vector/model/tensorflow.rb

@@ -32,7 +32,7 @@ class TensorFlowModel < VectorModel
       end
       @graph ||= keras_graph
       @graph.compile(**@compile_options)
-      @graph.fit(features, labels, :epochs => @epochs, :verbose => false)
+      @graph.fit(features, labels, :epochs => @epochs, :verbose => true)
       @graph.save(file)
     end
  

data/lib/rbbt/vector/model.rb

@@ -2,54 +2,85 @@ require 'rbbt/util/R'
 
 class VectorModel
   attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model
-  attr_accessor :features, :labels
+  attr_accessor :features, :names, :labels, :factor_levels
 
-  def self.R_run(model_file, features, labels, code)
+  def self.R_run(model_file, features, labels, code, names = nil, factor_levels = nil)
     TmpFile.with_file do |feature_file|
       Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
-      Open.write(feature_file + '.class', labels * "\n")
+      Open.write(feature_file + '.label', labels * "\n" + "\n")
+      Open.write(feature_file + '.names', names * "\n" + "\n") if names
+
+
+      what = case labels.first
+             when Numeric, Integer, Float
+               'numeric()'
+             else
+               'character()'
+             end
 
       R.run <<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
-labels = scan("#{ feature_file }.class");
-features = cbind(features, class = labels);
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=TRUE);
+#{"names(features) = make.names(readLines('#{feature_file + '.names'}'))" if names }
+#{ factor_levels.collect do |name,levels|
+    "features[['#{name}']] = factor(features[['#{name}']], levels=#{R.ruby2R levels})"
+  end * "\n" if factor_levels }
+labels = scan("#{ feature_file }.label", what=#{what});
+features = cbind(features, label = labels);
 #{code}
       EOF
     end
   end
 
-  def self.R_train(model_file, features, labels, code)
+  def self.R_train(model_file, features, labels, code, names = nil, factor_levels = nil)
     TmpFile.with_file do |feature_file|
       Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
-      Open.write(feature_file + '.class', labels * "\n")
+      Open.write(feature_file + '.label', labels * "\n" + "\n")
+      Open.write(feature_file + '.names', names * "\n" + "\n") if names
+
+      what = case labels.first
+             when Numeric, Integer, Float
+               'numeric()'
+             else
+               'character()'
+             end
 
       R.run <<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
-labels = scan("#{ feature_file }.class");
-features = cbind(features, class = labels);
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=TRUE);
+labels = scan("#{ feature_file }.label", what=#{what});
+#{"names(features) = make.names(readLines('#{feature_file + '.names'}'))" if names }
+features = cbind(features, label = labels);
+#{ factor_levels.collect do |name,levels|
+    "features[['#{name}']] = factor(features[['#{name}']], levels=#{R.ruby2R levels})"
+  end * "\n" if factor_levels }
 #{code}
 save(model, file='#{model_file}')
       EOF
     end
   end
 
-  def self.R_eval(model_file, features, list, code)
+  def self.R_eval(model_file, features, list, code, names = nil, factor_levels = nil)
     TmpFile.with_file do |feature_file|
+      if list
+        Open.write(feature_file, features.collect{|feat| feat * "\t"} * "\n" + "\n")
+      else
+        Open.write(feature_file, features * "\t" + "\n")
+      end
+      Open.write(feature_file + '.names', names * "\n" + "\n") if names
+
       TmpFile.with_file do |results|
-        if list
-          Open.write(feature_file, features.collect{|feat| feat * "\t"} * "\n" + "\n")
-        else
-          Open.write(feature_file, features * "\t" + "\n")
-        end
 
         io = R.run <<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=TRUE);
+#{"names(features) = make.names(readLines('#{feature_file + '.names'}'))" if names }
+#{ factor_levels.collect do |name,levels|
+    "features[['#{name}']] = factor(features[['#{name}']], levels=#{R.ruby2R levels})"
+  end * "\n" if factor_levels }
 load(file="#{model_file}");
 #{code}
 cat(paste(label, sep="\\n", collapse="\\n"));
         EOF
         txt = io.read
-        res = txt.sub(/WARNING: .*?\n/s,'').split(/\s+/).collect{|l| l.to_f}
+        res = txt.sub(/WARNING: .*?\n/s,'').split(/\s+/)
 
         if list
           res
@@ -66,7 +97,7 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     instance_eval code, file
   end
 
-  def initialize(directory, extract_features = nil, train_model = nil, eval_model = nil)
+  def initialize(directory, extract_features = nil, train_model = nil, eval_model = nil, names = nil, factor_levels = nil)
     @directory = directory
     FileUtils.mkdir_p @directory unless File.exists? @directory
 
@@ -76,6 +107,8 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     @eval_model_file = File.join(@directory, "eval_model")
     @train_model_file_R = File.join(@directory, "train_model.R")
     @eval_model_file_R = File.join(@directory, "eval_model.R")
+    @names_file = File.join(@directory, "feature_names")
+    @levels_file = File.join(@directory, "levels")
 
     if extract_features.nil?
       if File.exists?(@extract_features_file)
@@ -105,6 +138,22 @@ cat(paste(label, sep="\\n", collapse="\\n"));
       @eval_model = eval_model
     end
 
+    if names.nil?
+      if File.exists?(@names_file)
+        @names = Open.read(@names_file).split("\n")
+      end
+    else
+      @extract_features = names 
+    end
+
+    if factor_levels.nil?
+      if File.exists?(@levels_file)
+        @factor_levels = YAML.load(Open.read(@levels_file))
+      end
+    else
+      @factor_levels = factor_levels 
+    end
+
     @features = []
     @labels = []
   end
@@ -156,28 +205,31 @@ cat(paste(label, sep="\\n", collapse="\\n"));
     when String === eval_model
       Open.write(@eval_model_file_R, eval_model)
     end
+
+    Open.write(@levels_file, @factor_levels.to_yaml) if @factor_levels
+    Open.write(@names_file, @names * "\n" + "\n") if @names
   end
 
   def train
     case 
     when Proc === train_model
-      train_model.call(@model_file, @features, @labels)
+      train_model.call(@model_file, @features, @labels, @names, @factor_levels)
     when String === train_model
-      VectorModel.R_train(@model_file,  @features, @labels, train_model)
+      VectorModel.R_train(@model_file,  @features, @labels, train_model, @names, @factor_levels)
     end
     save_models
   end
 
   def run(code)
-    VectorModel.R_run(@model_file,  @features, @labels, code)
+    VectorModel.R_run(@model_file,  @features, @labels, code, @names, @factor_levels)
   end
 
   def eval(element)
     case 
     when Proc === @eval_model
-      @eval_model.call(@model_file, @extract_features.call(element), false)
+      @eval_model.call(@model_file, @extract_features.call(element), false, nil, @names, @factor_levels)
     when String === @eval_model
-      VectorModel.R_eval(@model_file,  @extract_features.call(element), false, eval_model)
+      VectorModel.R_eval(@model_file,  @extract_features.call(element), false, eval_model, @names, @factor_levels)
     end
   end
 
@@ -195,9 +247,9 @@ cat(paste(label, sep="\\n", collapse="\\n"));
 
     case 
     when Proc === eval_model
-      eval_model.call(@model_file, features, true)
+      eval_model.call(@model_file, features, true, nil, @names, @factor_levels)
     when String === eval_model
-      VectorModel.R_eval(@model_file, features, true, eval_model)
+      VectorModel.R_eval(@model_file, features, true, eval_model, @names, @factor_levels)
     end
   end
 
@@ -231,14 +283,61 @@ cat(paste(label, sep="\\n", collapse="\\n"));
 
   #  acc
   #end
+  #
+  
+  def self.f1_metrics(test, predicted, good_label = nil)
+    tp, tn, fp, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
+
+    labels = (test + predicted).uniq
+
+    if labels.length == 2 || good_label
+      good_label = labels.uniq.select{|l| l.to_s == "true"}.first if good_label.nil?
+      good_label = labels.uniq.select{|l| l.to_s == "1"}.first if good_label.nil?
+      good_label = labels.uniq.sort.first if good_label.nil?
+
+      test.zip(predicted).each do |gs,pred|
+        gs = gs.to_s
+        pred = pred.to_s
+
+        tp += 1 if gs == pred && gs == good_label
+        tn += 1 if gs == pred && gs != good_label 
+        fp += 1 if gs != good_label && pred == good_label
+        fn += 1 if gs == good_label && pred != good_label
+      end
+
+      p = tp + fn
+      pp = tp + fp
 
-  def cross_validation(folds = 10)
+      pr = tp.to_f / pp
+      re = tp.to_f / p
 
-    res = TSV.setup({}, "Fold~TP,TN,FP,FN,P,R,F1#:type=:list")
+      f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
+
+      [tp, tn, fp, fn, pr, re, f1]
+    else 
+      num = labels.length
+      acc = []
+      labels.each do |good_label|
+        values = VectorModel.f1_metrics(test, predicted, good_label)
+        acc << values
+      end
+      Misc.zip_fields(acc).collect{|s| Misc.mean(s)}
+    end
+  end
+
+  def cross_validation(folds = 10, good_label = nil)
 
     orig_features = @features
     orig_labels = @labels
 
+    multiclass = @labels.uniq.length > 2
+
+    if multiclass
+      res = TSV.setup({}, "Fold~P,R,F1#:type=:list")
+    else
+      res = TSV.setup({}, "Fold~TP,TN,FP,FN,P,R,F1#:type=:list")
+    end
+
     begin
       feature_folds = Misc.divide(@features, folds)
       labels_folds = Misc.divide(@labels, folds)
@@ -253,8 +352,6 @@ cat(paste(label, sep="\\n", collapse="\\n"));
         test_labels = labels_folds[fix]
         train_labels = labels_folds.values_at(*rest).flatten
 
-        tp, fp, tn, fn, pr, re, f1 = [0, 0, 0, 0, nil, nil, nil]
-
         @features = train_set
         @labels = train_labels
         self.train
@@ -262,26 +359,18 @@ cat(paste(label, sep="\\n", collapse="\\n"));
 
         raise "Number of predictions (#{predictions.length}) and test labels (#{test_labels.length}) do not match" if predictions.length != test_labels.length
 
-        test_labels.zip(predictions).each do |gs,pred|
-          gs = gs.to_i
-          pred = pred > 0.5 ? 1 : 0
-          tp += 1 if gs == pred && gs == 1
-          tn += 1 if gs == pred && gs == 0
-          fp += 1 if gs == 0 && pred == 1
-          fn += 1 if gs == 1 && pred == 0
-        end
-
-        p = tp + fn
-        pp = tp + fp
-
-        pr = tp.to_f / pp
-        re = tp.to_f / p
+        different_labels = test_labels.uniq
 
-        f1 = (2.0 * tp) / (2.0 * tp + fp + fn) 
+        tp, tn, fp, fn, pr, re, f1 = VectorModel.f1_metrics(test_labels, predictions, good_label)
 
-        Log.debug "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1} - #{[tp.to_s, tn.to_s, fp.to_s, fn.to_s] * " "}"
+        if multiclass 
+          Log.low "Multi-class CV Fold #{fix} - Average P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1}"
+          res[fix] = [pr,re,f1]
+        else
+          Log.low "CV Fold #{fix} P:#{"%.3f" % pr} R:#{"%.3f" % re} F1:#{"%.3f" % f1} - #{[tp.to_s, tn.to_s, fp.to_s, fn.to_s] * " "}"
+          res[fix] = [tp,tn,fp,fn,pr,re,f1]
+        end
 
-        res[fix] = [tp,tn,fp,fn,pr,re,f1]
       end
     ensure
       @features = orig_features

data/share/spaCy/cpu/textcat_accuracy.conf

@@ -0,0 +1,86 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = null
+
+[nlp]
+lang = "en"
+pipeline = ["tok2vec","textcat"]
+batch_size = 1000
+
+[components]
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["ORTH", "SHAPE"]
+rows = [5000, 2500]
+include_static_vectors = true
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 256
+depth = 8
+window_size = 1
+maxout_pieces = 3
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatEnsemble.v2"
+nO = null
+
+[components.textcat.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+[components.textcat.model.linear_model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+
+[initialize]
+vectors = "en_core_web_lg"

data/share/spaCy/cpu/textcat_efficiency.conf

@@ -0,0 +1,78 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = null
+
+[nlp]
+lang = "en"
+pipeline = ["tok2vec","textcat"]
+batch_size = 1000
+
+[components]
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["ORTH", "SHAPE"]
+rows = [5000, 2500]
+include_static_vectors = false
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+
+[initialize]
+vectors = null

data/share/spaCy/gpu/textcat_accuracy.conf

@@ -0,0 +1,84 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = "pytorch"
+
+[nlp]
+lang = "en"
+pipeline = ["transformer","textcat"]
+batch_size = 128
+
+[components]
+
+[components.transformer]
+factory = "transformer"
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v1"
+name = "emilyalsentzer/Bio_ClinicalBERT"
+tokenizer_config = {"use_fast": true}
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatEnsemble.v2"
+nO = null
+
+[components.textcat.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+
+[components.textcat.model.tok2vec.pooling]
+@layers = "reduce_mean.v1"
+
+[components.textcat.model.linear_model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 500
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 5e-5
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+
+[initialize]
+vectors = null

data/share/spaCy/gpu/textcat_efficiency.conf

@@ -0,0 +1,73 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = "pytorch"
+
+[nlp]
+lang = "en"
+pipeline = ["transformer","textcat"]
+batch_size = 128
+
+[components]
+
+[components.transformer]
+factory = "transformer"
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v1"
+name = "roberta-base"
+tokenizer_config = {"use_fast": true}
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 500
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 5e-5
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+
+[initialize]
+vectors = null

data/test/rbbt/vector/model/test_spaCy.rb

@@ -37,7 +37,7 @@ class TestSpaCyModel < Test::Unit::TestCase
       model.cross_validation
     end
 
-    def _test_svm_spacy
+    def test_svm_spacy
 
       require 'rbbt/tsv/csv'
       url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
@@ -84,5 +84,38 @@ class TestSpaCyModel < Test::Unit::TestCase
     end
   end
 
+  def test_spyCy_trf
+    TmpFile.with_file() do |dir|
+      Log.severity = 0
+      FileUtils.mkdir_p dir
+
+      model = SpaCyModel.new(
+        dir, 
+        "gpu/textcat_accuracy.conf"
+      )
+
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      good.keys[0..gsize-1].each do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      bad.keys[0..bsize-1].each do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+    end
+  end
 end
 

data/test/rbbt/vector/model/test_svm.rb

@@ -33,10 +33,10 @@ class TestSVMModel < Test::Unit::TestCase
 
       model.train
 
-      assert model.eval("1;1;1") > 0.5
-      assert model.eval("0;0;0") < 0.5
+      assert model.eval("1;1;1") == "1"
+      assert model.eval("0;0;0") == "0"
 
-      assert_equal [true, false], model.eval_list(%w(1;1;1 0;0;0)).collect{|v| v > 0.5}
+      assert_equal [true, false], model.eval_list(%w(1;1;1 0;0;0)).collect{|v| v == "1"}
     end
   end
 

data/test/rbbt/vector/model/test_tensorflow.rb

@@ -4,12 +4,13 @@ require 'rbbt/vector/model/tensorflow'
 class TestTensorflowModel < Test::Unit::TestCase
 
   def test_keras
+    Log.severity = 0
     TmpFile.with_file() do |dir|
       FileUtils.mkdir_p dir
 
       model = TensorFlowModel.new(
         dir, 
-        optimizer:'adam',
+        optimizer: 'adam',
         loss: 'sparse_categorical_crossentropy',
         metrics: ['accuracy']
       )
@@ -42,6 +43,7 @@ class TestTensorflowModel < Test::Unit::TestCase
 
         predictions = model.eval_list x_test.tolist()
         sum = 0
+
         predictions.zip(y_test.tolist()).each do |pred,label|
           sum += 1 if label.to_i == pred
         end
@@ -49,8 +51,6 @@ class TestTensorflowModel < Test::Unit::TestCase
       end
 
       assert sum.to_f / predictions.length > 0.7
-
-
     end
   end
 end

data/test/rbbt/vector/test_model.rb

@@ -211,7 +211,7 @@ cat(label, file="#{results}");
     end
   end
 
-  def test_model_list
+  def test_model_save
     text =<<-EOF
 1 0;1;1
 1 1;0;1
@@ -243,9 +243,9 @@ cat(label, file="#{results}");
           R.run <<-EOF
 features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
 labels = scan("#{ feature_file }.class", what=numeric());
-features = cbind(features, class = labels);
+features = cbind(features, label = labels);
 rbbt.require('e1071')
-model = svm(class ~ ., data = features) 
+model = svm(label ~ ., data = features) 
 save(model, file="#{ model_file }");
           EOF
         end
@@ -276,12 +276,18 @@ cat(label, file="#{results}");
 
       model.train
 
+      model = VectorModel.new(dir)
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        model.add features, label
+      end
+
       assert model.eval("1;1;1").to_f > 0.5
       assert model.eval("0;0;0").to_f < 0.5
     end
   end
 
-  def test_model_save
+  def test_model_name
     text =<<-EOF
 1 0;1;1
 1 1;0;1
@@ -298,50 +304,31 @@ cat(label, file="#{results}");
       FileUtils.mkdir_p dir
       model = VectorModel.new(dir)
 
-      model.extract_features = Proc.new{|element|
-        element.split(";")
-      }
+      model.names = %w(Var1 Var2 Var3)
 
-      model.train_model = Proc.new{|model_file,features,labels|
-        TmpFile.with_file do |feature_file|
-          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
-          Open.write(feature_file + '.class', labels * "\n")
-          R.run <<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
-labels = scan("#{ feature_file }.class", what=numeric());
-features = cbind(features, class = labels);
-rbbt.require('e1071')
-model = svm(class ~ ., data = features) 
-save(model, file="#{ model_file }");
-          EOF
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
         end
       }
 
-      model.eval_model = Proc.new{|model_file,features|
-        TmpFile.with_file do |feature_file|
-          TmpFile.with_file do |results|
-            Open.write(feature_file, features * "\t")
-            puts R.run(<<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+      model.train_model =<<-EOF
+rbbt.require('e1071')
+model = svm(as.factor(label) ~ Var1 + Var2, data = features) 
+      EOF
+
+      model.eval_model = <<-EOF
 library(e1071)
-load(file="#{ model_file }")
 label = predict(model, features);
-cat(label, file="#{results}");
-            EOF
-            ).read
-            Open.read(results)
-          end
-        end
-
-      }
+      EOF
 
       pairs = text.split(/\n/).collect do |line|
         label, features = line.split(" ")
-        [features, label]
+        model.add features, label
       end
 
-      model.add_list(*Misc.zip_fields(pairs))
-
       model.train
 
       assert model.eval("1;1;1").to_f > 0.5
@@ -349,23 +336,25 @@ cat(label, file="#{results}");
     end
   end
 
-  def test_model_save
+  def test_model_cv
     text =<<-EOF
-1 0;1;1
-1 1;0;1
-1 1;1;1
-1 0;1;1
-1 1;1;1
-0 0;1;0
-0 1;0;0
-0 0;1;0
-0 1;0;0
+0 0;1;0;0
+0 1;0;0;0
+0 0;1;0;0
+0 1;0;0;0
+1 0;1;1;0
+1 1;0;1;0
+1 1;1;1;0
+1 0;1;1;0
+1 1;1;1;0
     EOF
 
     TmpFile.with_file() do |dir|
       FileUtils.mkdir_p dir
       model = VectorModel.new(dir)
 
+      model.names = %w(Var1 Var2 Var3 Var4)
+
       model.extract_features = Proc.new{|element,list|
         if element
           element.split(";")
@@ -374,55 +363,151 @@ cat(label, file="#{results}");
         end
       }
 
-      model.train_model = Proc.new{|model_file,features,labels|
-        TmpFile.with_file do |feature_file|
-          Open.write(feature_file, features.collect{|feats| feats * "\t"} * "\n")
-          Open.write(feature_file + '.class', labels * "\n")
-          R.run <<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
-labels = scan("#{ feature_file }.class", what=numeric());
-features = cbind(features, class = labels);
-rbbt.require('e1071')
-model = svm(class ~ ., data = features) 
-save(model, file="#{ model_file }");
-          EOF
-        end
-      }
+      model.train_model =<<-EOF
+rbbt.require('randomForest')
+model = randomForest(as.factor(label) ~ ., data = features) 
+      EOF
 
-      model.eval_model = Proc.new{|model_file,features|
-        TmpFile.with_file do |feature_file|
-          TmpFile.with_file do |results|
-            Open.write(feature_file, features * "\t")
-            puts R.run(<<-EOF
-features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
-library(e1071)
-load(file="#{ model_file }")
+      model.eval_model = <<-EOF
+rbbt.require('randomForest')
 label = predict(model, features);
-cat(label, file="#{results}");
-            EOF
-            ).read
-            Open.read(results)
-          end
-        end
+      EOF
+
+      pairs = text.split(/\n/).collect do |line|
+        label, features = line.split(" ")
+        model.add features, label
+      end
+
+      model.train
+      
+      assert_equal "0", model.eval("1;1;0;0")
+      assert_equal "1", model.eval("1;1;1;0")
+
+      Log.with_severity 1 do
+        model.cross_validation(2)
+      end
 
+    end
+  end
+
+  def test_model_mclass
+    text =<<-EOF
+0 0;1;0;0
+0 1;0;0;0
+0 0;1;0;0
+0 1;0;0;0
+1 0;1;1;0
+1 1;0;1;0
+1 1;1;1;0
+1 0;1;1;0
+1 1;1;1;0
+2 0;1;0;1
+2 1;0;0;1
+2 1;1;0;1
+2 0;1;0;1
+2 1;1;0;1
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
+      model = VectorModel.new(dir)
+
+      model.names = %w(Var1 Var2 Var3 Var4)
+
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
+        end
       }
 
+      model.train_model =<<-EOF
+rbbt.require('randomForest')
+model = randomForest(as.factor(label) ~ ., data = features) 
+      EOF
+
+      model.eval_model = <<-EOF
+rbbt.require('randomForest')
+label = predict(model, features);
+      EOF
+
       pairs = text.split(/\n/).collect do |line|
         label, features = line.split(" ")
         model.add features, label
       end
 
       model.train
+      
+      assert_equal "0", model.eval("1;1;0;0")
+      assert_equal "1", model.eval("1;1;1;0")
+      assert_equal "2", model.eval("1;1;0;1")
+
+      Log.with_severity 1 do
+        model.cross_validation(2)
+      end
 
+    end
+  end
+
+  def test_model_factor_levels
+    text =<<-EOF
+0 0;1;0;f1
+0 1;0;0;f1
+0 0;1;0;f1
+0 1;0;0;f1
+1 0;1;1;f2
+1 1;0;1;f2
+1 1;1;1;f2
+1 0;1;1;f2
+1 1;1;1;f2
+    EOF
+
+    TmpFile.with_file() do |dir|
+      FileUtils.mkdir_p dir
       model = VectorModel.new(dir)
+
+      model.names = %w(Var1 Var2 Var3 Factor)
+
+      model.extract_features = Proc.new{|element,list|
+        if element
+          element.split(";")
+        elsif list
+          list.collect{|e| e.split(";") }
+        end
+      }
+
+      model.train_model =<<-EOF
+rbbt.require('randomForest')
+model = randomForest(as.factor(label) ~ ., data = features) 
+      EOF
+
+      model.eval_model = <<-EOF
+rbbt.require('randomForest')
+label = predict(model, features);
+      EOF
+
       pairs = text.split(/\n/).collect do |line|
         label, features = line.split(" ")
         model.add features, label
       end
 
-      assert model.eval("1;1;1").to_f > 0.5
-      assert model.eval("0;0;0").to_f < 0.5
+      Log.with_severity 0 do
+        model.train
+        model.cross_validation(2)
+
+        assert_raise do
+          assert_equal "0", model.eval("1;1;0;f1")
+        end
+
+        model.factor_levels = {"Factor" => %w(f1 f2)}
+        model.train
+        model = VectorModel.new(dir)
+        assert_equal "1", model.eval("1;1;1;f2")
+      end
+
     end
   end
 
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.50
+  version: 1.1.54
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-17 00:00:00.000000000 Z
+date: 2021-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -107,12 +107,17 @@ files:
 - lib/rbbt/statistics/rank_product.rb
 - lib/rbbt/tensorflow.rb
 - lib/rbbt/vector/model.rb
+- lib/rbbt/vector/model/random_forest.rb
 - lib/rbbt/vector/model/spaCy.rb
 - lib/rbbt/vector/model/svm.rb
 - lib/rbbt/vector/model/tensorflow.rb
 - share/R/MA.R
 - share/R/barcode.R
 - share/R/heatmap.3.R
+- share/spaCy/cpu/textcat_accuracy.conf
+- share/spaCy/cpu/textcat_efficiency.conf
+- share/spaCy/gpu/textcat_accuracy.conf
+- share/spaCy/gpu/textcat_efficiency.conf
 - test/rbbt/matrix/test_barcode.rb
 - test/rbbt/network/test_paths.rb
 - test/rbbt/statistics/test_fdr.rb