rbbt-util 4.1.0 → 4.2.0

data/lib/rbbt/util/named_array.rb

@@ -0,0 +1,113 @@
+require 'rbbt/util/chain_methods'
+require 'rbbt/util/misc'
+
+module NamedArray
+  extend ChainMethods
+
+  self.chain_prefix = :named_array
+  attr_accessor :fields
+  attr_accessor :key
+  attr_accessor :namespace
+
+  def self.setup(array, fields, key = nil, namespace = nil)
+    array.extend NamedArray unless NamedArray === array
+    array.fields = fields
+    array.key = key
+    array.namespace = namespace
+    array
+  end
+
+  def merge(array)
+    double = Array === array.first 
+    new = self.dup
+    (0..length - 1).each do |i|
+      if double
+        new[i] = new[i] + array[i]
+      else
+        new[i] << array[i]
+      end
+    end
+    new
+  end
+
+  def positions(fields)
+    if Array ==  fields
+      fields.collect{|field|
+        Misc.field_position(@fields, field)
+      }
+    else
+      Misc.field_position(@fields, fields)
+    end
+  end
+
+  def named_array_get_brackets(key)
+    pos = Misc.field_position(fields, key)
+    elem = named_array_clean_get_brackets(pos)
+
+    return elem if @fields.nil? or @fields.empty?
+
+    field = NamedArray === @fields ? @fields.named_array_clean_get_brackets(pos) : @fields[pos]
+    elem = Entity.formats[field].setup((elem.frozen? ? elem.dup : elem), (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include?(field) and not field == elem
+    elem
+  end
+
+  def named_array_each(&block)
+    if defined?(Entity) and not @fields.nil? and not @fields.empty?
+      @fields.zip(self).each do |field,elem|
+        elem = Entity.formats[field].setup((elem.frozen? ? elem.dup : elem), (namespace ? {:namespace => namespace, :organism => namespace} : {}).merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include?(field) and not field == elem
+        yield(elem)
+        elem
+      end
+    else
+      named_array_clean_each &block
+    end
+  end
+
+  def named_array_collect
+    res = []
+
+    named_array_each do |elem|
+      if block_given?
+        res << yield(elem)
+      else
+        res << elem
+      end
+    end
+
+    res
+  end
+
+  def named_array_set_brackets(key,value)
+    named_array_clean_set_brackets(Misc.field_position(fields, key), value)
+  end
+
+  def named_array_values_at(*keys)
+    keys = keys.collect{|k| Misc.field_position(fields, k) }
+    named_array_clean_values_at(*keys)
+  end
+
+  def zip_fields
+    return [] if self.empty?
+    zipped = Misc.zip_fields(self)
+    zipped = zipped.collect{|v| NamedArray.setup(v, fields)}
+    zipped 
+  end
+
+  def detach(file)
+    file_fields = file.fields.collect{|field| field.fullname}
+    detached_fields = []
+    self.fields.each_with_index{|field,i| detached_fields << i if file_fields.include? field.fullname}
+    fields = self.fields.values_at *detached_fields
+    values = self.values_at *detached_fields
+    values = NamedArray.name(values, fields)
+    values.zip_fields
+  end
+
+  def report
+    fields.zip(self).collect do |field,value|
+      "#{ field }: #{ Array === value ? value * "|" : value }"
+    end * "\n"
+  end
+
+end
+

data/lib/rbbt/util/open.rb

@@ -3,7 +3,6 @@ require 'rbbt/util/misc'
 require 'rbbt/util/tmpfile'
 
 require 'zlib'
-require 'digest/md5'
 
 module Open
   class OpenURLError < StandardError; end
@@ -77,12 +76,14 @@ module Open
     end
   end
 
+  def self.digest_url(url, options = {})
+    params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
+    digest = Misc.digest(params.inspect)
+  end
   # Cache
   #
   def self.in_cache(url, options = {})
-    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
-
-    filename = File.join(REMOTE_CACHEDIR, digest)
+    filename = File.join(REMOTE_CACHEDIR, digest_url(url, options))
     if File.exists? filename
       return filename 
     else
@@ -91,7 +92,7 @@ module Open
   end
  
   def self.remove_from_cache(url, options = {})
-    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
+    digest = Misc.digest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
 
     filename = File.join(REMOTE_CACHEDIR, digest)
     if File.exists? filename
@@ -102,8 +103,10 @@ module Open
   end
   
   def self.add_cache(url, data, options = {})
-    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
-    Misc.sensiblewrite(File.join(REMOTE_CACHEDIR, digest), data)
+    file = File.join(REMOTE_CACHEDIR, digest_url(url, options))
+    Misc.lock(file) do
+      Misc.sensiblewrite(file, data)
+    end
   end
 
   # Grep
@@ -168,16 +171,20 @@ module Open
     wget_options[:nice_key] = options.delete(:nice_key)
     wget_options[:quiet] = options.delete(:quiet)
     wget_options["--post-data="] = options.delete(:post) if options.include? :post
+    wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
+    wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
     wget_options[:cookies] = options.delete(:cookies)
 
     io = case
          when (not remote?(url))
            file_open(url, options[:grep])
-         when options[:nocache]
+         when (options[:nocache] and options[:nocache] != :update)
            # What about grep?
            wget(url, wget_options)
-         when in_cache(url, wget_options)
-           file_open(in_cache(url, wget_options), options[:grep])
+         when (options[:nocache] != :update and in_cache(url, wget_options))
+           Misc.lock(in_cache(url, wget_options)) do
+             file_open(in_cache(url, wget_options), options[:grep])
+           end
          else
            io = wget(url, wget_options)
            add_cache(url, io, wget_options)

data/lib/rbbt/workflow/accessor.rb

@@ -26,7 +26,7 @@ class Step
     Misc.lock(info_file) do
       i = info
       i[key] = value
-      Open.write(info_file, i.to_yaml)
+      Open.write(info_file, YAML.dump(i))
       value
     end
   end
@@ -140,6 +140,14 @@ class Step
 end
 
 module Workflow
+  def log(status, message = nil)
+    if message
+      Log.low "#{ status }: #{ message }"
+    else
+      Log.low "#{ status }"
+    end
+  end
+
   def task_info(name)
     task = tasks[name]
     description = task.description

data/lib/rbbt/workflow/step.rb

@@ -23,7 +23,7 @@ class Step
     @inputs = inputs || []
   end
 
-  def prepare_entity(value, description = nil, info = {})
+  def prepare_result(value, description = nil, info = {})
     return value if description.nil?
     Entity.formats[description].setup(value, info.merge(:format => description)) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? description
     value
@@ -31,7 +31,7 @@ class Step
 
   def exec
     result = @task.exec_in self, *@inputs
-    prepare_entity result, @task.result_description
+    prepare_result result, @task.result_description
   end
 
   def join
@@ -41,17 +41,18 @@ class Step
         sleep 5
       end
     else
+      Log.debug "Waiting for pid: #{@pid}"
       Process.waitpid @pid
       @pid = nil
     end
     self
   end
 
-  def run
-    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq do
+  def run(no_load = false)
+    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
       log task.name, "Starting task: #{ name }"
       set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
-      @dependencies.each{|dependency| dependency.run}
+      @dependencies.each{|dependency| dependency.run true}
       set_info :status, :start
       set_info :inputs, Misc.zip2hash(task.inputs, @inputs) unless task.inputs.nil?
       res = exec
@@ -59,7 +60,7 @@ class Step
       res
     end
 
-    prepare_entity result, @task.result_description, info
+    prepare_result result, @task.result_description, info
   end
 
   def fork
@@ -90,7 +91,7 @@ class Step
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
       exec
     end
-    prepare_entity result, @task.result_description, info
+    prepare_result result, @task.result_description, info
   end
 
   def clean
@@ -108,6 +109,7 @@ class Step
   def rec_dependencies
     @dependencies.collect{|step| step.rec_dependencies}.flatten.concat  @dependencies
   end
+
   def step(name)
     rec_dependencies.select{|step| step.task.name.to_sym == name.to_sym}.first
   end

data/share/lib/R/util.R

@@ -1,7 +1,11 @@
-rbbt.ruby <- function(code, load = TRUE){
+rbbt.ruby <- function(code, load = TRUE, flat = FALSE){
   file = system('rbbt_exec.rb - file', input = code, intern=TRUE);
   if (load){
-    data = rbbt.tsv(file);
+    if(flat){
+        data = rbbt.flat.tsv(file);
+    }else{
+        data = rbbt.tsv(file);
+    }
     rm(file);
     return(data);
   }else{
@@ -25,8 +29,26 @@ rbbt.load.data <- function(filename, sep = "\t",  ...){
   return(data);
 }
 
+rbbt.flat.tsv <- function(filename, sep = "\t", comment.char ="#", ...){
+  f = file(filename, 'r');
+  headers = readLines(f, 1);
+  if (length(grep("^#: ", headers)) > 0){ 
+      headers = readLines(f, 1); 
+  } 
+  result = list();
+  while( TRUE ){
+    line = readLines(f, 1);
+    if (length(line) == 0){ break;}
+    parts = unlist(strsplit(line, sep, fixed = TRUE));
+    id = parts[1];
+    result[[id]] = parts[2:length(parts)];
+  }
+  close(f);
+  return(result);
+}
+
 rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1,  ...){
-  data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, quote='', row.names= row.names, comment.char = comment.char, ...);
+  data=read.table(file=filename, sep=sep, fill=TRUE, as.is=TRUE, quote='', row.names= row.names, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
   if (length(grep("^#: ", headers)) > 0){ 
@@ -148,8 +170,8 @@ rbbt.run <- function(filename){
 
 # Addapted from http://www.phaget4.org/R/image_matrix.html
 rbbt.plot.matrix <- function(x, ...){
-    min <- min(x);
-    max <- max(x);
+    min <- min(x, na.rm=T);
+    max <- max(x, na.rm=T);
     yLabels <- rownames(x);
     xLabels <- colnames(x);
     title <-c();
@@ -184,7 +206,7 @@ rbbt.plot.matrix <- function(x, ...){
     ColorRamp <- rgb( seq(0,1,length=256),  # Red
                       seq(0,1,length=256),  # Green
                       seq(1,0,length=256))  # Blue
-        ColorLevels <- seq(min, max, length=length(ColorRamp));
+    ColorLevels <- seq(min, max, length=length(ColorRamp));
 
 # Reverse Y axis
     reverse <- nrow(x) : 1;

data/test/rbbt/test_annotations.rb

@@ -35,7 +35,7 @@ class TestAnnotations < Test::Unit::TestCase
   def test_array
     ary = ["string"]
     annotation_str = "Annotation String"
-    AnnotatedArray.setup_chain(ary)
+    ary.extend AnnotatedArray
     AnnotatedString.setup(ary, annotation_str)
     assert_equal [AnnotatedString], ary.annotation_types
     assert_equal annotation_str, ary.annotation_str
@@ -87,6 +87,20 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal str1, Annotated.load_tsv(Annotated.tsv([str1, str2], :literal, :JSON)).sort.first
   end
 
+  def test_load_array_tsv
+    str1 = "string1"
+    str2 = "string2"
+    a = [str1, str2]
+    annotation_str = "Annotation String 2"
+    AnnotatedString.setup(a, annotation_str)
+    a.extend AnnotatedArray
+
+
+    assert_equal annotation_str, Annotated.load_tsv(Annotated.tsv(a, :all)).annotation_str
+
+    assert_equal str1, Annotated.load_tsv(Annotated.tsv(a, :literal, :JSON)).sort.first
+  end
+
   def test_inheritance
     str = "string1"
     annotation_str1 = "Annotation String 1"
@@ -104,7 +118,7 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal str + annotation_str, str.add_annot
   end
 
- def test_annotation_positional2hash
+  def test_annotation_positional2hash
     str = "string"
     annotation_str = "Annotation String"
     AnnotatedString.setup(str, :annotation_str => annotation_str)

data/test/rbbt/test_persist.rb

@@ -3,7 +3,43 @@ require 'rbbt/persist'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
+module TestAnnotation
+  extend Annotation
+
+  self.annotation :test_annotation
+end
 class TestPersist < Test::Unit::TestCase
+
+  def test_annotation_persist
+    TmpFile.with_file do |tmp|
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+
+      annotations = [entity1, entity2]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
   def test_array_persist
     TmpFile.with_file do |tmp|
       10.times do
@@ -12,7 +48,7 @@ class TestPersist < Test::Unit::TestCase
         end)
       end
     end
-  
+
     TmpFile.with_file do |tmp|
       10.times do
         assert_equal [],(Persist.persist("Test", :array, :file => tmp) do

data/test/rbbt/test_tsv.rb

@@ -458,5 +458,17 @@ row2    A|AA|AAA
  
   end
 
+  def test_flat2single
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a aa aaa
+row2    A AA AAA
+    EOF
 
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value").include? "aaa"
+    end
+ 
+  end
 end

data/test/rbbt/tsv/test_attach.rb

@@ -221,7 +221,7 @@ row2    E
 
     tsv1.attach tsv2, :fields => ["OtherID"], :persist_input => true
     
-    assert_equal tsv1.fields,%w(ValueA ValueB OtherID)
+    assert_equal tsv1.fields, %w(ValueA ValueB OtherID)
     assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
   end