rbbt-dm 1.1.47 → 1.1.52

data/lib/rbbt/vector/model/svm.rb

@@ -9,7 +9,7 @@ class SVMModel < VectorModel
 
     @train_model =<<-EOF
 library(e1071);
-model = svm(class ~ ., data = features, scale=c(0));
+model = svm(as.factor(class) ~ ., data = features);
     EOF
  
     @eval_model =<<-EOF

data/lib/rbbt/vector/model/tensorflow.rb

@@ -0,0 +1,55 @@
+require 'rbbt/vector/model'
+require 'rbbt/tensorflow'
+
+class TensorFlowModel < VectorModel
+  attr_accessor :graph, :epochs, :compile_options
+
+  def tensorflow(&block)
+    RbbtPython.run "tensorflow" do 
+      RbbtPython.module_eval(&block)
+    end
+  end
+
+  def keras(&block)
+    RbbtPython.run "tensorflow.keras", as: 'keras' do 
+      RbbtPython.run "tensorflow" do 
+        RbbtPython.module_eval(&block)
+      end
+    end
+  end
+  
+  def initialize(dir, graph = nil, epochs = 3, **compile_options)
+    @graph = graph
+    @epochs = epochs
+    @compile_options = compile_options
+
+    super(dir)
+
+    @train_model = Proc.new do |file, features, labels|
+      tensorflow do 
+        features = tensorflow.convert_to_tensor(features)
+        labels = tensorflow.convert_to_tensor(labels)
+      end
+      @graph ||= keras_graph
+      @graph.compile(**@compile_options)
+      @graph.fit(features, labels, :epochs => @epochs, :verbose => false)
+      @graph.save(file)
+    end
+ 
+    @eval_model = Proc.new do |file, features|
+      tensorflow do 
+        features = tensorflow.convert_to_tensor(features)
+      end
+      keras do
+        @graph ||= keras.models.load_model(file)
+        indices = @graph.predict(features, :verbose => false).tolist()
+        labels = indices.collect{|p| p.length > 1 ? p.index(p.max): p.first }
+        labels
+      end
+    end
+  end
+
+  def keras_graph(&block)
+    @graph = keras(&block)
+  end
+end

data/share/spaCy/cpu/textcat_accuracy.conf

@@ -0,0 +1,86 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = null
+
+[nlp]
+lang = "en"
+pipeline = ["tok2vec","textcat"]
+batch_size = 1000
+
+[components]
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["ORTH", "SHAPE"]
+rows = [5000, 2500]
+include_static_vectors = true
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 256
+depth = 8
+window_size = 1
+maxout_pieces = 3
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatEnsemble.v2"
+nO = null
+
+[components.textcat.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+[components.textcat.model.linear_model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+
+[initialize]
+vectors = "en_core_web_lg"

data/share/spaCy/cpu/textcat_efficiency.conf

@@ -0,0 +1,78 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = null
+
+[nlp]
+lang = "en"
+pipeline = ["tok2vec","textcat"]
+batch_size = 1000
+
+[components]
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["ORTH", "SHAPE"]
+rows = [5000, 2500]
+include_static_vectors = false
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+
+[initialize]
+vectors = null

data/share/spaCy/gpu/textcat_accuracy.conf

@@ -0,0 +1,84 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = "pytorch"
+
+[nlp]
+lang = "en"
+pipeline = ["transformer","textcat"]
+batch_size = 128
+
+[components]
+
+[components.transformer]
+factory = "transformer"
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v1"
+name = "roberta-base"
+tokenizer_config = {"use_fast": true}
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatEnsemble.v2"
+nO = null
+
+[components.textcat.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+
+[components.textcat.model.tok2vec.pooling]
+@layers = "reduce_mean.v1"
+
+[components.textcat.model.linear_model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 500
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 5e-5
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+
+[initialize]
+vectors = null

data/share/spaCy/gpu/textcat_efficiency.conf

@@ -0,0 +1,73 @@
+# This is an auto-generated partial config. To use it with 'spacy train'
+# you can run spacy init fill-config to auto-fill all default settings:
+# python -m spacy init fill-config ./base_config.cfg ./config.cfg
+[paths]
+train = null
+dev = null
+
+[system]
+gpu_allocator = "pytorch"
+
+[nlp]
+lang = "en"
+pipeline = ["transformer","textcat"]
+batch_size = 128
+
+[components]
+
+[components.transformer]
+factory = "transformer"
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v1"
+name = "roberta-base"
+tokenizer_config = {"use_fast": true}
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.textcat]
+factory = "textcat"
+
+[components.textcat.model]
+@architectures = "spacy.TextCatBOW.v1"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+
+[corpora]
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 500
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 5e-5
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+
+[initialize]
+vectors = null

data/test/rbbt/matrix/test_barcode.rb

@@ -11,7 +11,7 @@ class TestBarcode < Test::Unit::TestCase
     data["G4"] = [6,6,1,1,1,1]
 
     TmpFile.with_file(data.to_s) do |file|
-      m = Matrix.new file
+      m = RbbtMatrix.new file
       m.barcode(file+'.barcode')
       tsv =  TSV.open(file+'.barcode')
       assert tsv["G2"] = [0,1,0,1,0,1]

data/test/rbbt/network/test_paths.rb

@@ -20,7 +20,7 @@ N4 N5
     end_node = "N5"
 
     path = Paths.dijkstra(network, start_node, [end_node])
-    assert_equal %w(N1 N2 N4), path.reverse
+    assert_equal %w(N1 N2 N4 N5), path.reverse
   end
 
   def test_weighted_dijsktra

data/test/rbbt/vector/model/test_spaCy.rb

@@ -0,0 +1,121 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/vector/model/spaCy'
+require 'rbbt/vector/model/svm'
+
+class TestSpaCyModel < Test::Unit::TestCase
+
+  def test_spyCy
+    TmpFile.with_file() do |dir|
+      Log.severity = 0
+      FileUtils.mkdir_p dir
+
+      model = SpaCyModel.new(
+        dir, 
+        "cpu/textcat_efficiency.conf"
+      )
+
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      good.keys[0..gsize-1].each do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      bad.keys[0..bsize-1].each do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+    end
+
+    def test_svm_spacy
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      model = SVMModel.new(
+        dir
+      )
+
+      nlp = RbbtPython.run "spacy" do
+        spacy.load('en_core_web_md')
+      end
+
+      model.extract_features = Proc.new do |text|
+        vs = RbbtPython.run do
+          RbbtPython.collect nlp.(text).__iter__ do |token|
+            token.vector.tolist()
+          end
+        end
+        length = vs.length
+
+        v = vs.inject(nil){|acc,ev| acc = acc.nil? ? ev : acc.zip(ev).collect{|a,b| a + b } }
+
+        v.collect{|e| e / length }
+      end
+
+      TSV.traverse good.keys[0..gsize-1], :type => :array, :bar => true do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      TSV.traverse bad.keys[0..bsize-1], :type => :array, :bar => true do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+
+    end
+  end
+
+  def test_spyCy_trf
+    TmpFile.with_file() do |dir|
+      Log.severity = 0
+      FileUtils.mkdir_p dir
+
+      model = SpaCyModel.new(
+        dir, 
+        "gpu/textcat_accuracy.conf"
+      )
+
+
+      require 'rbbt/tsv/csv'
+      url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
+      tsv = TSV.csv(Open.open(url))
+      tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+
+      good = tsv.select("Recommended IND" => '1')
+      bad = tsv.select("Recommended IND" => '0')
+
+      gsize = 2000
+      bsize = 500
+      good.keys[0..gsize-1].each do |text|
+        next if text.nil? || text.empty?
+        model.add text, '1'
+      end
+
+      bad.keys[0..bsize-1].each do |text|
+        model.add text, '0'
+      end
+
+      model.cross_validation
+    end
+  end
+end
+