rbbt-util 3.1.0 → 3.2.0

data/lib/rbbt/util/tsv/manipulate.rb

@@ -12,7 +12,7 @@ class TSV
                           when Integer === new_fields
                             [new_fields]
                           when String === new_fields
-                            [identify_field new_fields]
+                            [identify_field(new_fields)]
                           when Array === new_fields
                             new_fields.collect{|new_field| identify_field new_field}
                           when new_fields == :key
@@ -46,7 +46,7 @@ class TSV
 
     # Cycle through
     if monitor
-      desc = "Parsing Stream"
+      desc = "Iterating TSV"
       step = 100
       if Hash === monitor
         desc = monitor[:desc] if monitor.include? :desc 
@@ -84,14 +84,22 @@ class TSV
                              f
                            else
                              f = fields.dup
-                             if type == :double
+                             case
+                             when type == :single
+                               f = [f,key]
+                             when type == :double
                                f.push [key]
                              else
                                f.push key
                              end
                              f.values_at *new_field_positions.collect{|pos| pos == :key ? -1 : pos }
                            end
-        new_field_values = NamedArray.name new_field_values, new_field_names
+        
+        if type == :single
+          new_field_values = new_field_values.first
+        else
+          new_field_values = NamedArray.name new_field_values, new_field_names unless unnamed
+        end
 
         next if new_key_value.nil? or (String === new_key_value and new_key_value.empty?)
         yield new_key_value, new_field_values
@@ -221,14 +229,17 @@ class TSV
         method.each{|item| new[item] = self[item] if self.include? item}
       when Array === method
         through :key, key do |key, values|
+          values = [values] if type == :single
           new[key] = self[key] if (values.flatten & method).any?
         end
       when Regexp === method
         through :key, key do |key, values|
+          values = [values] if type == :single
           new[key] = self[key] if values.flatten.select{|v| v =~ method}.any?
         end
       when String === method
         through :key, key do |key, values|
+          values = [values] if type == :single
           new[key] = self[key] if values.flatten.select{|v| v == method}.any?
         end
       end
@@ -267,7 +278,7 @@ class TSV
   end
 
   def add_field(name = nil)
-    each do |key, values|
+    through do |key, values|
       new_values = yield(key, values)
       new_values = [new_values] if type == :double and not Array === new_values
 
@@ -280,7 +291,7 @@ class TSV
   end
 
   def add_fields(names = nil)
-    each do |key, values|
+    through do |key, values|
       new_values = yield(key, values)
       new_values = [new_values] if type == :double and not Array == new_values
 

data/lib/rbbt/util/tsv/misc.rb

@@ -29,3 +29,13 @@ class TSV
     counts
   end
 end
+
+module Open
+  def self.tsv(file, *args)
+    TSV.new file, *args
+  end
+
+  def self.index(file, *args)
+    TSV.index file, *args
+  end
+end

data/lib/rbbt/util/tsv/parse.rb

@@ -142,12 +142,29 @@ class TSV
 
     #{{{ Process rest
     data = options[:persistence_data] || {}
+    if Persistence::TSV === data
+      serializer = case
+                   when ((cast == "to_i" or cast == :to_i) and type == :single)
+                     :integer 
+                   when ((cast == "to_i" or cast == :to_i) and (type == :flat or type == :list))
+                     :integer_array 
+                   when (type == :list or type == :flat)
+                     :list
+                   when type == :single
+                     :single
+                   else
+                     :double
+                   end
+      data.serializer = serializer
+    end
+
+
     single = type.to_sym != :double
     max_cols = 0
     while line do
       line.chomp!
 
-      progress_monitor.tick(stream.pos) if progress_monitor
+      progress_monitor.tick(stream.pos) if progress_monitor 
 
       if line.empty?                           or
          (exclude and     exclude.call(line))  or

data/lib/rbbt/util/workflow.rb

@@ -4,6 +4,15 @@ require 'rbbt/util/persistence'
 require 'rbbt/util/misc'
 
 module WorkFlow
+
+  def self.require_workflow(path)
+    if Rbbt.etc.workflow_dir.exists?
+      require Dir.glob(File.join(Rbbt.etc.workflow_dir.read.strip, '*', path + '.rb')).first
+    else
+      require Dir.glob(File.join(Rbbt.share.workflows.find, '*', path + '.rb')).first
+    end
+  end
+
   def self.extended(base)
     class << base
       attr_accessor :tasks, :jobdir, :dangling_options, :dangling_option_descriptions,
@@ -53,8 +62,8 @@ module WorkFlow
     @dangling_option_defaults[name] = default if default
   end
 
-  def task_dependencies(dependencies)
-    dependencies = [dependencies] unless Array === dependencies
+  def task_dependencies(*dependencies)
+    dependencies = dependencies.flatten
     @dangling_dependencies = dependencies.collect{|dep| Symbol === dep ? tasks[dep] : dep }
   end
 
@@ -93,7 +102,7 @@ module WorkFlow
     @last_task = task
   end
 
-  def job(task, jobname, *args)
+  def job(task, jobname = "Default", *args)
     tasks[task].job(jobname, *args)
   end
 

data/lib/rbbt/util/workflow/soap.rb

@@ -105,7 +105,6 @@ if __FILE__ == $0
   class SequenceWF < WorkFlowWS
     self.klass = Organism
   end
-
   
   wf = SequenceWF.new
   wf.export :genomic_mutations_to_genes

data/share/install/software/lib/install_helpers

@@ -8,8 +8,6 @@ OPT_SCM_DIR="$SOFTWARE_DIR/scm"; [ -d $OPT_SCM_DIR ] || mkdir -p $OPT_SCM_DIR
 OPT_JAR_DIR="$OPT_DIR/jars"; [ -d $OPT_JAR_DIR ] || mkdir -p $OPT_JAR_DIR
 OPT_BUILD_DIR="$SOFTWARE_DIR/.build"; [ -d $OPT_BUILD_DIR ] || mkdir -p $OPT_BUILD_DIR
 
-#source "$HOME/config/bash/_utility_functions"
-
 function expand_path(){
   name=$(basename $1)
   dir=$(dirname $1)

data/share/lib/R/util.R

@@ -26,7 +26,7 @@ rbbt.load.data <- function(filename, sep = "\t",  ...){
 }
 
 rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1,  ...){
-  data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, row.names=row.names, comment.char = comment.char, ...);
+  data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, row.names= row.names, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
   if (length(grep("^#: ", headers)) > 0){ 
@@ -93,7 +93,7 @@ rbbt.sort_by_field <- function(data, field, is.numeric=TRUE){
 
 rbbt.add <- function(data, new){
     if (is.null(data)){
-        return(new);
+        return(c(new));
     }else{
         return(c(data, new));
     }
@@ -101,7 +101,7 @@ rbbt.add <- function(data, new){
 
 rbbt.acc <- function(data, new){
     if (is.null(data)){
-        return(new);
+        return(c(new));
     }else{
         return(unique(c(data, new)));
     }

data/test/rbbt/util/test_cmd.rb

@@ -22,6 +22,7 @@ class TestCmd < Test::Unit::TestCase
   end
 
   def test_pipe
+    assert_equal("test\n", CMD.cmd("echo test", :pipe => true).read)
     assert_equal("test\n", CMD.cmd("echo '{opt}' test", :pipe => true).read)
     assert_equal("test", CMD.cmd("echo '{opt}' test", "-n" => true, :pipe => true).read)
     assert_equal("test2\n", CMD.cmd("cut", "-f" => 2, "-d" => '" "', :in => "test1 test2", :pipe => true).read)
@@ -41,4 +42,26 @@ class TestCmd < Test::Unit::TestCase
     assert_raise CMD::CMDError do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).read end
   end
 
+  def test_pipes
+    text = <<-EOF
+line1
+line2
+line3
+line11
+line22
+line33
+    EOF
+
+    TmpFile.with_file(text * 100) do |file|
+      CMD.cmd("gzip #{ file }")
+
+      gz = CMD.cmd("gunzip", :in => File.open(file + '.gz'), :pipe => true)
+      io = CMD.cmd('tail -n 10', :in => gz, :pipe => true)
+      assert_equal 10, io.read.split(/\n/).length
+
+
+    end
+
+  end
+
 end

data/test/rbbt/util/test_excel2tsv.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/excel2tsv'
 
 class TestTSV < Test::Unit::TestCase
   def test_tsv2excel
-    tsv = TSV.excel2tsv(test_datafile('Test.xls'), :header => true)
+    tsv = TSV.excel2tsv(datafile_test('Test.xls'), :header => true)
     assert_equal 'Id', tsv.key_field
   end
 end

data/test/rbbt/util/test_misc.rb

@@ -5,7 +5,7 @@ require 'test/unit'
 class TestMisc < Test::Unit::TestCase
 
   def test_pdf2text_example
-    assert PDF2Text.pdf2text(test_datafile('example.pdf')).read =~ /An Example Paper/i
+    assert PDF2Text.pdf2text(datafile_test('example.pdf')).read =~ /An Example Paper/i
   end
 
   def test_pdf2text_EPAR
@@ -13,7 +13,7 @@ class TestMisc < Test::Unit::TestCase
   end
 
   def test_pdf2text_wrong
-    assert_raise CMD::CMDError do PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB#") end
+    assert_raise CMD::CMDError do PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB#").read end
   end
 
   def test_string2hash
@@ -84,32 +84,62 @@ This is an example file. Entries are separated by Entry
     a = {:a => 1, "b" => 2}
     a.extend IndiferentHash
 
-    assert 1, a["a"]
-    assert 1, a[:a]
-    assert 2, a["b"]
-    assert 2, a[:b]
+    assert_equal 1, a["a"]
+    assert_equal 1, a[:a]
+    assert_equal 2, a["b"]
+    assert_equal 2, a[:b]
   end
 
   def test_lockfile
+
     TmpFile.with_file do |tmpfile|
       pids = []
-      3.times do |i|
+      4.times do |i|
         pids << Process.fork do 
-          pid = pid.to_s
-          Misc.lock(tmpfile, pid) do |f, val|
+          pid = Process.pid().to_s
+          status = Misc.lock(tmpfile, pid) do |f, val|
             Open.write(f, val)
             sleep rand * 2
             if pid == Open.read(tmpfile)
-              exit(0)
+              0
             else
-              exit(1)
+              1
             end
           end
+          exit(status)
         end
+
       end
       pids.each do |pid| Process.waitpid pid; assert $?.success? end
     end
 
   end
 
+  def test_divide
+    assert_equal 2, Misc.divide(%w(1 2 3 4 5 6 7 8 9),2).length
+  end
+
+  def test_process_to_hash
+    list = [1,2,3,4]
+    assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
+  end
+
+  def test_add_method
+    a = "Test"
+    Misc.add_method a, :invert do self.reverse end
+    assert_equal "Test".reverse, a.invert
+  end
+
+  def test_redefine_method
+    a = "Test"
+    worked = false
+    Misc.redefine_method a, :reverse, :old_reverse do worked = true; self.old_reverse end
+    assert_equal "Test".reverse, a.reverse
+    assert worked
+  end
+
+  def test_merge_sorted_arrays
+    assert_equal [1,2,3,4], Misc.merge_sorted_arrays([1,3], [2,4])
+  end
+
 end

data/test/rbbt/util/test_open.rb

@@ -49,7 +49,7 @@ class TestOpen < Test::Unit::TestCase
     end
   end
 
-   def test_read_grep
+  def test_read_grep
     content =<<-EOF
 1
 2
@@ -67,7 +67,7 @@ class TestOpen < Test::Unit::TestCase
       Open.read(file, :grep => ["1","3"]) do |line| sum += line.to_i end
       assert_equal(1 + 3, sum)
     end
- 
+
   end
 
   def test_gzip

data/test/rbbt/util/test_persistence.rb

@@ -153,11 +153,11 @@ row2   2 4 6 8
   end
 
   def test_non_blocking
-    $a = TSV.new test_datafile('test.tsv'), :persistence => true, :persistence_dir => Rbbt.tmp.test.persistence
+    $a = TSV.new datafile_test('test.tsv'), :persistence => true, :persistence_dir => Rbbt.tmp.test.persistence
     $a.data.read
 
     pid = Process.fork do
-      $b = TSV.new test_datafile('test.tsv'), :persistence => true, :persistence_dir => Rbbt.tmp.test.persistence
+      $b = TSV.new datafile_test('test.tsv'), :persistence => true, :persistence_dir => Rbbt.tmp.test.persistence
       $b.data.close
     end
 

data/test/rbbt/util/test_resource.rb

@@ -13,18 +13,6 @@ file 'foo' do |t|
 end
   EOF
 
-  tmp.work.define_as_rake tmp.Rakefile.find.produce
-
-  tmp.test.install.xclip.define_as_string <<-EOF
-name="xclip:0.12"
-url="http://downloads.sourceforge.net/project/xclip/xclip/0.12/xclip-0.12.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fxclip%2F&ts=1286472387&use_mirror=sunet"
-
-install_src "$name" "$url"
-  EOF
-
-  FileUtils.chmod 0770, tmp.test.install.xclip.produce
-
-  software.opt.xclip.define_as_install tmp.test.install.xclip.find
 end
 
 Open.cachedir = Rbbt.tmp.cache.find :user
@@ -36,10 +24,10 @@ end
 
 class TestResource < Test::Unit::TestCase
   def test_methods
-    assert Resource.methods.include?("resources")
-    assert ! Resource.methods.include?("pkgdir")
-    assert ! Phgx.methods.include?("resources")
-    assert Phgx.methods.include?("pkgdir")
+    assert Resource.methods.collect{|m| m.to_s}.include?("resources") 
+    assert ! Resource.methods.collect{|m| m.to_s}.include?("pkgdir")
+    assert ! Phgx.methods.collect{|m| m.to_s}.include?("resources")
+    assert Phgx.methods.collect{|m| m.to_s}.include?("pkgdir")
 
   end
   def test_resolve
@@ -84,9 +72,5 @@ class TestResource < Test::Unit::TestCase
       FileUtils.rm Rbbt.tmp.url.find if File.exists? Rbbt.tmp.url.find
     end
   end
-
-  def test_install
-    assert File.exists?(Rbbt.software.opt.xclip.produce)
-  end
 end
 

data/test/rbbt/util/test_tc_hash.rb

@@ -86,5 +86,43 @@ class TestTCHash < Test::Unit::TestCase
       Process.wait pid
     end
   end
+
+  def test_serializer_alias
+    TmpFile.with_file do |f|
+      t = TCHash.get f, true, :double
+      t["1"] = [[1],[2]]
+      t["2"] = [[3],[4,5]]
+
+      t = TCHash.get f
+      assert_equal [["3"],["4","5"]], t["2"]
+
+      t.close
+      TCHash::CONNECTIONS.clear
+
+      t = TCHash.get f
+      assert_equal [["3"],["4","5"]], t["2"]
+    end
+  end
+
+  def test_serializer_reload
+    TmpFile.with_file do |f|
+      t = TCHash.get f, true, :double
+      t["1"] = [[1],[2]]
+      t["2"] = [[3],[4,5]]
+
+      t = TCHash.get f
+      assert_equal TCHash::StringDoubleArraySerializer, t.serializer
+      assert_equal [["3"],["4","5"]], t["2"]
+
+      t.close
+      TCHash::CONNECTIONS.clear
+
+      t = TCHash.get f
+      assert_equal [["3"],["4","5"]], t["2"]
+    end
+  end
+
+
+
 end