rbbt-util 3.0.3 → 3.1.0

data/lib/rbbt/util/tsv/manipulate.rb

@@ -1,5 +1,7 @@
 class TSV
 
+  attr_accessor :monitor
+ 
   def through(new_key_field = :key, new_fields = nil, &block)
 
     # Get positions
@@ -43,11 +45,23 @@ class TSV
                       end if fields
 
     # Cycle through
+    if monitor
+      desc = "Parsing Stream"
+      step = 100
+      if Hash === monitor
+        desc = monitor[:desc] if monitor.include? :desc 
+        step = monitor[:step] if monitor.include? :step 
+      end
+      progress_monitor = Progress::Bar.new(size, 0, step, desc)
+    else
+      progress_monitor = nil
+    end
                       
     if new_key_position == :key and ( new_fields.nil? or new_fields == fields)
-      each do |key, fields| yield key, fields end
+      each do |key, fields| progress_monitor.tick if progress_monitor; yield key, fields end
     else
       each do |key, fields|
+        progress_monitor.tick if progress_monitor;
         new_key_value = case
                         when (new_key_position.nil? or new_key_position == :key)
                           key
@@ -190,8 +204,14 @@ class TSV
         new[key] = values if [key,values].flatten.select{|v| v =~ method}.any?
       end
     when String === method
-      through do |key, values|
-        new[key] = values if [key,values].flatten.select{|v| v == method}.any?
+      if block_given?
+        through do |key, values|
+          new[key] = values if yield((method == key_field or method == :key)? key : values[method])
+        end
+      else
+        through do |key, values|
+          new[key] = values if [key,values].flatten.select{|v| v == method}.any?
+        end
       end
     when Hash === method
       key  = method.keys.first
@@ -255,6 +275,8 @@ class TSV
     end
 
     self.fields = self.fields + [name] if fields != nil and name != nil
+
+    self
   end
 
   def add_fields(names = nil)

data/lib/rbbt/util/tsv/misc.rb

@@ -0,0 +1,31 @@
+require 'rbbt/util/tsv'
+
+class TSV
+  def self.keys(file, sep = "\t")
+    CMD.cmd("cut -f 1 -d'#{sep}' '#{file}'|grep -v ^#").read.split("\n")
+  end
+
+  def self.field_match_counts(file, values)
+    key_field, fields = TSV.parse_header(Open.open(file))
+    fields.unshift key_field
+
+    counts = {}
+    TmpFile.with_file do |tmpfile|
+      if Array === values
+        Open.write(tmpfile, values * "\n")
+      else
+        FileUtils.ln_s values, tmpfile
+      end
+
+      fields.each_with_index do |field,i|
+        counts[field] = begin
+                          CMD.cmd("cat #{ file } |grep -v ^#|cut -f #{i + 1}|tr '|' '\\n' |sort -u |grep [[:alpha:]]|grep -f #{tmpfile} -F -w").read.count("\n")
+                        rescue
+                          0
+                        end
+      end
+    end
+
+    counts
+  end
+end

data/lib/rbbt/util/tsv/parse.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/misc'
+require 'progress-bar'
 class TSV
  
   def self.parse_fields(io, delimiter = "\t")
@@ -7,7 +8,6 @@ class TSV
     ## split with delimiter, do not remove empty
     fields = io.split(delimiter, -1)
 
-
     fields
   end
 
@@ -58,6 +58,7 @@ class TSV
     options = Misc.add_defaults options, more_options
 
     options = Misc.add_defaults options, 
+      :monitor          => false,
       :case_insensitive => false,
       :type             => :double,
       :namespace        => nil,
@@ -78,6 +79,8 @@ class TSV
       :exclude          => nil,
       :select           => nil,
       :grep             => nil
+
+    monitor = Misc.process_options options, :monitor
     
     header_hash, sep, sep2 =
       Misc.process_options options, :header_hash, :sep, :sep2
@@ -119,13 +122,33 @@ class TSV
 
     exclude ||= Misc.process_options options, :reject if options.include? :reject
 
+    if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
+      size = case
+             when stream.respond_to?(:size)
+               stream.size
+             else
+               stream.stat.size
+             end
+      desc = "Parsing Stream"
+      step = 100
+      if Hash === monitor
+        desc = monitor[:desc] if monitor.include? :desc 
+        step = monitor[:step] if monitor.include? :step 
+      end
+      progress_monitor = Progress::Bar.new(size, 0, step, desc)
+    else
+      progress_monitor = nil
+    end
+
     #{{{ Process rest
-    data = {}
+    data = options[:persistence_data] || {}
     single = type.to_sym != :double
     max_cols = 0
     while line do
       line.chomp!
 
+      progress_monitor.tick(stream.pos) if progress_monitor
+
       if line.empty?                           or
          (exclude and     exclude.call(line))  or
          (select  and not select.call(line))
@@ -187,7 +210,7 @@ class TSV
             data[id] = extra.first
         when type == :flat
           if data.include? id
-            data[id].concat extra
+            data[id] = data[id] + extra
           else
             data[id] = extra 
           end
@@ -261,7 +284,7 @@ class TSV
       end
     end
 
-    if keep_empty and max_cols > 0
+    if keep_empty and max_cols > 0 and not Persistence::TSV === data
       data.each do |key, values| 
         next if values =~ /__Ref:/
         new_values = values

data/lib/rbbt/util/tsv/resource.rb

@@ -16,6 +16,12 @@ module Resource
       namespace || File.basename(File.dirname(self))
     end
 
+    def to_yaml(opts = {})
+      YAML.quick_emit( nil, opts ) { |out|
+        out.scalar( taguri, self, :plain )
+      }
+    end
+
     def index(options = {})
       TSV.index self, options
     end

data/lib/rbbt/util/workflow.rb

@@ -55,7 +55,7 @@ module WorkFlow
 
   def task_dependencies(dependencies)
     dependencies = [dependencies] unless Array === dependencies
-    @dangling_dependencies = dependencies
+    @dangling_dependencies = dependencies.collect{|dep| Symbol === dep ? tasks[dep] : dep }
   end
 
   def task_description(description)

data/lib/rbbt/util/workflow/soap.rb

@@ -0,0 +1,117 @@
+require 'simplews'
+require 'rbbt/util/workflow'
+require 'base64'
+
+class WorkFlowWS < SimpleWS
+
+  def self.klass=(klass)
+    @klass = klass
+  end
+
+  def self.klass
+    @klass || self
+  end
+
+  def task(name)
+    self.class.klass.tasks[name]
+  end
+
+  def export(name)
+    task = self.class.klass.tasks[name]
+
+    options, optional_options = task.option_summary
+
+    desc task.description
+    options.each do |option|
+      param_desc option[:name] => option[:description] if option[:description]
+    end
+    param_desc :return => "Job Identifier"
+    option_names = [:name] + options.collect{|option| option[:name]}
+    option_types = Hash[*option_names.zip([ :string] + options.collect{|option| option[:type] || :string}).flatten]
+    serve name, option_names, option_types do |*args|
+      task(name).job(*args).fork.id
+    end
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  serve :abort, %w(task id), {:return => false, :task => :string, :id => :string} do |task, id|
+    task(task).load(id).abort
+    nil
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "Info hash in yaml"
+  serve :info, %w(task id), {:task => :string, :id => :string} do |task, id|
+    Open.read(task(task).load(id).info_file)
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "Step"
+  serve :step, %w(task id), {:task => :string, :id => :string} do |task, id|
+    task(task).load(id).step.to_s
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "True if job is done (error or not)"
+  serve :done, %w(task id), {:return => :boolean, :task => :string, :id => :string} do |task, id|
+    task(task).load(id).done?
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "True if job finished with error. Error message is the last of the messages (see 'messages' method)."
+  serve :error, %w(task id), {:return => :boolean, :task => :string, :id => :string} do |task, id|
+    task(task).load(id).error?
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "Messages"
+  serve :messages, %w(task id), {:return => :array, :task => :string, :id => :string} do |task, id|
+    task(task).load(id).messages
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "Job result in Base64"
+  serve :load, %w(task id), {:return => :binary, :task => :string, :id => :string} do |task, id|
+    Base64.encode64(task(task).load(id).read)
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :return => "File names"
+  serve :files, %w(task id), {:return => :array, :task => :string, :id => :string} do |task, id|
+    task(task).load(id).files
+  end
+
+  param_desc :task => "Task name"
+  param_desc :id   => "Job id"
+  param_desc :file   => "File name"
+  param_desc :return => "File contents in Base64"
+  serve :file, %w(task id file), {:return => :array, :task => :string, :id => :string, :file => :string} do |task, id|
+    Base64.encode64(task(task).load(id).files(file).read)
+  end
+end
+
+
+if __FILE__ == $0
+
+  require 'rbbt/sources/organism/sequence'
+  class SequenceWF < WorkFlowWS
+    self.klass = Organism
+  end
+
+  
+  wf = SequenceWF.new
+  wf.export :genomic_mutations_to_genes
+  Open.write('/tmp/foo.wsdl', wf.wsdl)
+  wf.start
+end
+
+
+

data/share/lib/R/util.R

@@ -1,3 +1,14 @@
+rbbt.ruby <- function(code, load = TRUE){
+  file = system('rbbt_exec.rb - file', input = code, intern=TRUE);
+  if (load){
+    data = rbbt.tsv(file);
+    rm(file);
+    return(data);
+  }else{
+    return(file);
+  }
+}
+
 rbbt.glob <- function(d, pattern){
     d=sub("/$", '', d);
     sapply(dir(d, pattern), function(file){paste(d,file,sep="/")});
@@ -14,8 +25,8 @@ rbbt.load.data <- function(filename, sep = "\t",  ...){
   return(data);
 }
 
-rbbt.tsv <- function(filename, sep = "\t", comment.char ="#",  ...){
-  data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, row.names=1, comment.char = comment.char, ...);
+rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1,  ...){
+  data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, row.names=row.names, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
   if (length(grep("^#: ", headers)) > 0){ 
@@ -30,6 +41,13 @@ rbbt.tsv <- function(filename, sep = "\t", comment.char ="#",  ...){
   return(data);
 }
 
+rbbt.tsv2matrix <- function(data){
+  new <- data.matrix(data);
+  colnames(new) <- colnames(data);
+  rownames(new) <- rownames(data);
+  return(new);
+}
+
 rbbt.tsv.write <- function(filename, data, key.field = NULL){
   if (is.null(key.field)){ key.field = "ID";}
 
@@ -55,6 +73,14 @@ rbbt.percent <- function(values){
     values=values/sum(values);
 }
 
+rbbt.split <- function(string){
+  return(unlist(strsplit(string, "\\|")));
+}
+
+rbbt.last <-function(data){
+  data[length(data)];
+}
+
 rbbt.sort_by_field <- function(data, field, is.numeric=TRUE){
     if (is.numeric){
         field.data=as.numeric(data[,field]);
@@ -89,3 +115,27 @@ rbbt.init <- function(data, new){
     }
 }
 
+rbbt.this.script = system("rbbt_Rutil.rb", intern =T)
+
+rbbt.reload <- function (){
+    source(rbbt.this.script)
+}
+
+rbbt.parse <- function(filename){
+    f <- file(filename, open='r');
+    lines <- readLines(f);
+    close(f);
+
+    from = match(1,as.vector(sapply(lines, function(x){grep('#[[:space:]]*START',x,ignore.case=TRUE)})));
+    to   = match(1,as.vector(sapply(lines, function(x){grep('#[[:space:]]*END',x,ignore.case=TRUE)})));
+    if (is.na(from)){from = 1}
+    if (is.na(to)){to = length(lines)}
+    return(parse(text=paste(lines[from:to],sep="\n")));
+}
+
+rbbt.run <- function(filename){
+    rbbt.reload();
+    eval(rbbt.parse(filename), envir=globalenv());
+}
+
+

data/test/rbbt/util/test_misc.rb

@@ -4,19 +4,19 @@ require 'test/unit'
 
 class TestMisc < Test::Unit::TestCase
 
-  def _test_pdf2text_example
+  def test_pdf2text_example
     assert PDF2Text.pdf2text(test_datafile('example.pdf')).read =~ /An Example Paper/i
   end
 
-  def _test_pdf2text_EPAR
+  def test_pdf2text_EPAR
     assert PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB/document_library/EPAR_-_Scientific_Discussion/human/000402/WC500033103.pdf").read =~ /Tamiflu/i
   end
 
-  def _test_pdf2text_wrong
+  def test_pdf2text_wrong
     assert_raise CMD::CMDError do PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB#") end
   end
 
-  def _test_string2hash
+  def test_string2hash
     assert(Misc.string2hash("--user-agent=firefox").include? "--user-agent")
     assert(Misc.string2hash(":true")[:true] == true)
     assert(Misc.string2hash("true")["true"] == true)
@@ -27,17 +27,17 @@ class TestMisc < Test::Unit::TestCase
     assert(Misc.string2hash("a=b#c=d#:h=:j")[:h] == :j)
   end
   
-  def _test_named_array
+  def test_named_array
     a = NamedArray.name([1,2,3,4], %w(a b c d))
     assert_equal(1, a['a'])
   end
 
-  def _test_path_relative_to
+  def test_path_relative_to
     assert_equal "test/foo", Misc.path_relative_to('test/test/foo', 'test')
   end
 
-  def _test_chunk
-    _test =<<-EOF
+  def test_chunk
+    test =<<-EOF
 This is an example file. Entries are separated by Entry
 -- Entry
 1
@@ -52,7 +52,7 @@ This is an example file. Entries are separated by Entry
     assert_equal "1\n2\n3", Misc.chunk(test, /^-- Entry/).first.strip
   end
 
-  def _test_hash2string
+  def test_hash2string
     hash = {}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
 
@@ -73,14 +73,14 @@ This is an example file. Entries are separated by Entry
  
  end
 
-  def _test_merge
+  def test_merge
     a = [[1],[2]]
     a = NamedArray.name a, %w(1 2)
     a.merge [3,4]
     assert_equal [1,3], a[0]
   end
 
-  def _test_indiferent_hash
+  def test_indiferent_hash
     a = {:a => 1, "b" => 2}
     a.extend IndiferentHash