rbbt-util 4.2.0 → 4.3.0

data/lib/rbbt/annotations.rb

@@ -13,6 +13,7 @@ module Annotated
   end
 
   def annotations
+    raise "Annotation types is nil for object: #{self.inspect}" if annotation_types.nil?
     annotation_types.collect do |mod|
       mod.annotations
     end.flatten.uniq
@@ -32,9 +33,11 @@ module Annotated
   end
 
   def self.load(object, info)
-    annotation_types = info[:annotation_types]
+    annotation_types = info[:annotation_types] || []
     annotation_types = annotation_types.split("+") if String === annotation_types
 
+    return object if annotation_types.nil? or annotation_types.empty?
+
     annotation_types.each do |mod|
       mod = Misc.string2const(mod) if String === mod
       mod.setup(object, *info.values_at(*mod.all_annotations))
@@ -59,10 +62,8 @@ module Annotated
           info.to_json
         when field == "annotation_types"
           annotation_types.collect{|t| t.to_s} * "+"
-        when field == "literal array"
-          (self * "|").gsub(/\n|\t/, ' ')
         when field == "literal"
-          self.gsub(/\n|\t/, ' ')
+          (Array === self ? "Array:" << self * "|" : self).gsub(/\n|\t/, ' ')
         when info.include?(field.to_sym)
           info.delete(field.to_sym)
         when self.respond_to?(field)
@@ -78,17 +79,16 @@ module Annotated
     fields = fields.flatten
     info = {}
     literal_pos = fields.index "literal"
-    literal_array_pos = fields.index "literal array"
 
     object = case
              when literal_pos
                values[literal_pos]
-             when literal_array_pos
-               values[literal_array_pos].split("|").extend AnnotatedArray
              else
                id.dup
              end
 
+    object = object["Array:".length..-1].split("|") if object =~ /^Array:/
+
     if Array === values
       Misc.zip_fields(values).collect do |list|
         fields.each_with_index do |field,i|
@@ -120,23 +120,23 @@ module Annotated
     return nil if annotations.nil?
     fields = case
              when ((fields.compact.empty?) and not annotations.empty?)
-               fields = AnnotatedArray === annotations ? annotations.annotations : annotations.first.annotations
+               fields = AnnotatedArray === annotations ? annotations.annotations : annotations.compact.first.annotations
                fields << :annotation_types
              when (fields == [:literal] and not annotations.empty?)
                fields << :literal
              when (fields == [:all] and not annotations.empty?)
-               fields = [:annotation_types] + (Annotated === annotations ? annotations.annotations : annotations.first.annotations)
+               fields = [:annotation_types] + (Annotated === annotations ? annotations.annotations : annotations.compact.first.annotations)
                fields << :literal
+             when annotations.empty?
+               [:annotation_types, :literal]
              else
                fields.flatten
              end
 
     fields = fields.collect{|f| f.to_s}
 
-    fields = fields.collect{|f| ((f == "literal" and AnnotatedArray === annotations) ? "literal array" : f)}
-
     case
-    when (Annotated === annotations and not annotations.double_array)
+    when (Annotated === annotations and not (AnnotatedArray === annotations and annotations.double_array))
       tsv = TSV.setup({}, :key_field => "Single", :fields => fields, :type => :list, :unnamed => true)
       tsv[annotations.id] = annotations.tsv_values(*fields)
     when Array === annotations 
@@ -157,10 +157,11 @@ module Annotated
         Annotated.load_tsv_values(id, values, tsv.fields)
       end
 
-      if tsv.key_field == "Single"
+      case tsv.key_field 
+      when "Single"
         annotated_entities.first
       else
-        annotated_entities[0].annotate annotated_entities unless annotated_entities.empty?
+        annotated_entities
       end
     end
   end
@@ -260,10 +261,11 @@ end
 
 module AnnotatedArray
   extend ChainMethods
+
   self.chain_prefix = :annotated_array
 
   def double_array
-    AnnotatedArray === self.first
+    AnnotatedArray === self.annotated_array_clean_get_brackets(0)
   end
 
   def annotated_array_first
@@ -276,6 +278,7 @@ module AnnotatedArray
 
   def annotated_array_get_brackets(pos)
     value = annotated_array_clean_get_brackets(pos)
+    value = value.dup if value.frozen?
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.all_annotations))
     end
@@ -303,8 +306,14 @@ module AnnotatedArray
   def annotated_array_collect
     res = []
 
-    annotated_array_each do |value|
-      res << yield(value)
+    if block_given?
+      annotated_array_each do |value|
+        res << yield(value)
+      end
+    else
+      annotated_array_each do |value|
+        res << value
+      end
     end
 
     res
@@ -352,11 +361,14 @@ module AnnotatedArray
 
   def annotated_array_remove(list)
     value = (self - list)
+
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+
     value.context = self.context
     value.container = self.container
+
     value
   end
 
@@ -369,6 +381,7 @@ module AnnotatedArray
 
     value.context = self.context
     value.container = self.container
+
     value
   end
  
@@ -381,8 +394,10 @@ module AnnotatedArray
 
     value.context = self.context
     value.container = self.container
+
     value
   end
+
   def annotated_array_flatten
     value = self.annotated_array_clean_flatten.dup
 
@@ -392,27 +407,33 @@ module AnnotatedArray
 
     value.context = self.context
     value.container = self.container
+
     value
   end
 
   def annotated_array_reverse
     value = self.annotated_array_clean_reverse
+
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+
     value.context = self.context
     value.container = self.container
+
     value
   end
 
-
   def annotated_array_sort_by(&block)
     value = self.annotated_array_clean_sort_by &block
+
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+
     value.context = self.context
     value.container = self.container
+
     value
   end
 
@@ -422,8 +443,10 @@ module AnnotatedArray
     annotation_types.each do |mod|
       mod.setup(value, *info.values_at(*mod.annotations))
     end
+
     value.context = self.context
     value.container = self.container
+
     value
   end
 
@@ -445,4 +468,9 @@ module AnnotatedArray
 
     value
   end
+
+  def self.annotate(list)
+    list[0].annotate list unless AnnotatedArray === list or list[0].nil? or (not list[0].respond_to? :annotate)
+  end
+
 end

data/lib/rbbt/persist.rb

@@ -155,9 +155,72 @@ module Persist
     if persist_options[:persist]
       path = persistence_path(name, persist_options, other_options || {})
 
-      case type
-      when :memory
+      case 
+      when type.to_sym === :memory
         Persist::MEMORY[path] ||= yield
+
+      when (type.to_sym == :annotations and persist_options.include? :annotation_repo)
+
+        repo = persist_options[:annotation_repo]
+
+        keys = nil
+        subkey = name + ":"
+
+        if String === repo
+          repo = Persist.open_tokyocabinet(repo, false, :list, "BDB")
+          repo.read_and_close do
+            keys = repo.range subkey + 0.chr, true, subkey + 254.chr, true
+          end
+          repo.close
+        else
+          repo.read_and_close do
+            keys = repo.range subkey + 0.chr, true, subkey + 254.chr, true
+          end
+        end
+
+        case
+        when (keys.length == 1 and keys.first == subkey + 'NIL')
+          nil
+        when (keys.length == 1 and keys.first == subkey + 'EMPTY')
+          []
+        when (keys.length == 1 and keys.first =~ /:SINGLE$/)
+          key = keys.first
+          values = repo.write_and_close do
+            repo[key]
+          end
+          Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
+        when keys.any?
+          repo.read_and_close do
+            keys.collect{|key|
+              v = repo[key]
+              Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
+            }
+          end
+        else
+          entities = yield
+
+          Misc.lock(repo.persistence_path) do
+            repo.write_and_close do 
+              case
+              when entities.nil?
+                repo[subkey + "NIL"] = nil
+              when entities.empty?
+                repo[subkey + "EMPTY"] = nil
+              when (not Array === entities or AnnotatedArray === entities)
+                tsv_values = entities.tsv_values("literal", "annotation_types", "JSON") 
+                repo[subkey + entities.id << ":" << "SINGLE"] = tsv_values
+              else
+                entities.each do |e|
+                  tsv_values = e.tsv_values("literal", "annotation_types", "JSON") 
+                  repo[subkey + e.id] = tsv_values
+                end
+              end
+            end
+          end
+
+          entities
+        end
+
       else
         Misc.lock(path) do
           if is_persisted?(path, persist_options)
@@ -167,11 +230,18 @@ module Persist
           else
             Log.debug "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
           end
-          res = yield
-          save_file(path, type, res)
-          res
+          begin
+            res = yield
+            save_file(path, type, res)
+            res
+          rescue
+            Log.high "Error in persist. Erasing '#{ path }'"
+            FileUtils.rm path if File.exists? path
+            raise $!
+          end
         end
       end
+
     else
       yield
     end

data/lib/rbbt/persist/tsv.rb

@@ -2,13 +2,18 @@ require 'tokyocabinet'
 
 module Persist
   TC_CONNECTIONS = {}
-  def self.open_tokyocabinet(path, write, serializer = nil)
+
+  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
     write = true if not File.exists?(path)
-    flags = (write ? TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT : TokyoCabinet::HDB::OREADER)
+
+    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB"
+    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+
+    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
 
     FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
 
-    database = TC_CONNECTIONS[path] ||= TokyoCabinet::HDB.new
+    database = TC_CONNECTIONS[path] ||= tokyocabinet_class.new
     database.close
 
     if !database.open(path, flags)
@@ -18,7 +23,11 @@ module Persist
 
     if not database.respond_to? :old_close
       class << database
-        attr_accessor :writable, :closed, :persistence_path
+        attr_accessor :writable, :closed, :persistence_path, :tokyocabinet_class
+
+        def closed?
+          @closed
+        end
 
         alias old_close close
         def close
@@ -29,7 +38,7 @@ module Persist
         def read(force = false)
           return if not write? and not closed and not force
           self.close
-          if !self.open(@persistence_path, TokyoCabinet::BDB::OREADER)
+          if !self.open(@persistence_path, tokyocabinet_class::OREADER)
             ecode = self.ecode
             raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
           end
@@ -41,10 +50,12 @@ module Persist
         def write(force = true)
           return if write? and not closed and not force
           self.close
-          if !self.open(@persistence_path, TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT)
+
+          if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
             ecode = self.ecode
             raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
           end
+
           @writable = true
           @closed = false
           self
@@ -70,6 +81,19 @@ module Persist
           out(key)
         end
 
+        def write_and_close
+          write if @closed or not write?
+          res = yield
+          close
+          res
+        end
+
+        def read_and_close
+          read if @closed or write?
+          res = yield
+          close
+          res
+        end
 
         def merge!(hash)
           hash.each do |key,values|
@@ -77,10 +101,19 @@ module Persist
           end
         end
 
+        if instance_methods.include? "range"
+          alias old_range range
+
+          def range(*args)
+            keys = old_range(*args)
+            keys - TSV::ENTRY_KEYS
+          end
+        end
       end
     end
 
     database.persistence_path ||= path
+    database.tokyocabinet_class = tokyocabinet_class
 
     TSV.setup database
     database.serializer = serializer || database.serializer 
@@ -103,7 +136,7 @@ module Persist
 
              if is_persisted? path
                Log.debug "TSV persistence up-to-date: #{ path }"
-               return open_tokyocabinet(path, false) 
+               return Misc.lock(path) do open_tokyocabinet(path, false); end
              else
                Log.debug "TSV persistence creating: #{ path }"
              end
@@ -112,13 +145,24 @@ module Persist
 
              data = open_tokyocabinet(path, true, persist_options[:serializer])
              data.serializer = :type unless data.serializer
+
+             data.close
+
              data
            else
-             data = {}
+             {}
            end
 
     begin
-      yield data
+      if data.respond_to? :persistence_path and data != persist_options[:data]
+        Misc.lock data.persistence_path do
+          data.write_and_close do
+            yield data
+          end
+        end
+      else
+        yield data
+      end
     rescue Exception
       begin
         data.close if data.respondo_to? :close
@@ -128,7 +172,7 @@ module Persist
       raise $!
     end
 
-    data.read if data.respond_to? :read and  data.respond_to? :write? and data.write?
+    data.read if data.respond_to? :read and ((data.respond_to?(:write?) and data.write?) or (data.respond_to?(:closed?) and data.closed?))
 
     data
   end

data/lib/rbbt/resource.rb

@@ -66,33 +66,42 @@ module Resource
       raise "Resource #{ path } does not seem to be claimed"
     end
 
-    case type
-    when :string
-      Open.write(path, content)
-    when :url
-      Open.write(path, Open.open(content))
-    when :proc
-      data = content.call
-      Open.write(path, data) unless File.exists? path
-    when :rake
-      run_rake(path, content, rake_dir)
-    when :install
-      Log.debug "Installing software: #{path}"
-      software_dir = path.resource.root.software.find :user
-      preamble = <<-EOF
+    if not File.exists? path
+      Misc.lock path + '.produce' do
+        begin
+          case type
+          when :string
+            Open.write(path, content)
+          when :url
+            Open.write(path, Open.open(content))
+          when :proc
+            data = content.call
+            Open.write(path, data) 
+          when :rake
+            run_rake(path, content, rake_dir)
+          when :install
+            Log.debug "Installing software: #{path}"
+            software_dir = path.resource.root.software.find :user
+            preamble = <<-EOF
 #!/bin/bash
 
 RBBT_SOFTWARE_DIR="#{software_dir}"
 
 INSTALL_HELPER_FILE="#{Rbbt.share.install.software.lib.install_helpers.find :lib, caller_lib_dir(__FILE__)}"
 source "$INSTALL_HELPER_FILE"
-      EOF
+            EOF
 
-      CMD.cmd('bash', :in => preamble + "\n" + Open.read(content))
+            CMD.cmd('bash', :in => preamble + "\n" + Open.read(content))
 
-      set_software_env(software_dir)
-    else
-      raise "Could not produce #{ resource }. (#{ type }, #{ content })"
+            set_software_env(software_dir)
+          else
+            raise "Could not produce #{ resource }. (#{ type }, #{ content })"
+          end
+        rescue
+          FileUtils.rm path if File.exists? path
+          raise $!
+        end
+      end
     end
 
     path

data/lib/rbbt/resource/path.rb

@@ -59,9 +59,12 @@ module Path
     :global => File.join('/', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
     :local => File.join('/usr/local', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
     :lib => File.join('{LIBDIR}', "{TOPLEVEL}", "{SUBPATH}"),
+    :default => :user
   }
 
-  def find(where = nil, caller_lib = nil)
+  def find(where = nil, caller_lib = nil, search_paths = nil)
+    where = search_paths[:default] if where == :default
+    search_paths ||= SEARCH_PATHS
     return self if located?
     if self.match(/(.*?)\/(.*)/)
       toplevel, subpath = self.match(/(.*?)\/(.*)/).values_at 1, 2
@@ -71,15 +74,19 @@ module Path
 
     path = nil
     if where.nil?
-      SEARCH_PATHS.keys.each do |w| 
-        path = find(w, caller_lib)
+      search_paths.keys.each do |w| 
+        path = find(w, caller_lib, search_paths)
         return path if File.exists? path
       end
-      find :user
+      if search_paths.include? :default
+        find((search_paths[:default] || :user), caller_lib, search_paths)
+      else
+        raise "Path '#{ path }' not found, and no default specified in search paths: #{search_paths.inspect}"
+      end
     else
       libdir = where == :lib ? Path.caller_lib_dir(caller_lib) : ""
       libdir ||= ""
-      Path.setup SEARCH_PATHS[where].sub('{PKGDIR}', pkgdir).sub('{TOPLEVEL}', toplevel).sub('{SUBPATH}', subpath).sub('{LIBDIR}', libdir), @pkgdir, @resource
+      Path.setup search_paths[where].sub('{PKGDIR}', pkgdir).sub('{TOPLEVEL}', toplevel).sub('{SUBPATH}', subpath).sub('{LIBDIR}', libdir), @pkgdir, @resource
     end
   end
 

data/lib/rbbt/tsv/accessor.rb

@@ -306,11 +306,17 @@ if '#{entry}' == 'serializer'
 
         define_method :serialized_get do |key|
           return nil unless self.include? key
-          self.serializer_module.load(tsv_clean_get_brackets(key))
+          res = tsv_clean_get_brackets(key)
+          return res if res.nil?
+          self.serializer_module.load(res)
         end
 
         define_method :serialized_set do |key, value|
-          tsv_clean_set_brackets key, self.serializer_module.dump(value)
+          if value.nil?
+            tsv_clean_set_brackets key, value
+          else
+            tsv_clean_set_brackets key, self.serializer_module.dump(value)
+          end
         end
       end
     end

data/lib/rbbt/tsv/parser.rb

@@ -258,6 +258,7 @@ module TSV
           @key_position = key_field
         when String === key_field
           @key_position = @fields.dup.unshift(@key_field).index key_field
+          raise "Key field #{ key_field } was not found" if @key_position.nil?
         else
           raise "Format of key_field not understood: #{key_field.inspect}"
         end

data/lib/rbbt/tsv/util.rb

@@ -87,6 +87,7 @@ module TSV
     when (field.nil? or field == :key or key_field == field)
       :key
     when String === field
+      raise "No fields specified in TSV.identify_field" if fields.nil?
       pos = fields.index field
       Log.medium "Field #{ field } was not found. Options: #{fields * ", "}" if pos.nil?
       pos

data/lib/rbbt/util/cmd.rb

@@ -54,13 +54,17 @@ module CMD
     def force_close
       if @pid
         Log.debug "Forcing close by killing '#{@pid}'" if log
-        Process.kill("KILL", @pid)
-        Process.waitpid(@pid)
+        begin
+          Process.kill("KILL", @pid)
+          Process.waitpid(@pid)
+        rescue
+          Log.low "Exception in forcing close of command [#{ @pid }, #{cmd}]: #{$!.message}"
+        end
       end
 
       @post.call if @post
 
-      original_close
+      original_close unless self.closed?
     end
 
     def read(*args)

data/lib/rbbt/util/log.rb

@@ -8,14 +8,23 @@ module Log
   WARN     = 5
   ERROR    = 6
 
-  def self.severity=(severity)
-    @@severity = severity
+  class << self
+    attr_accessor :logfile, :severity
   end
 
-  def self.severity
-    @@severity
+
+  def self.logfile
+    @logfile = nil
   end
 
+  #def self.severity=(severity)
+  #  @severity = severity
+  #end
+
+  #def self.severity
+  #  @severity
+  #end
+
   SEVERITY_COLOR = ["0;37m", "0;32m", "0;33m", "0;31m", "1;0m" ].collect{|e| "\033[#{e}"}
 
   def self.log(message, severity = MEDIUM)
@@ -23,9 +32,11 @@ module Log
     severity_color = SEVERITY_COLOR[severity]
     font_color = {false => "\033[0;37m", true => "\033[0m"}[severity >= INFO]
 
-    STDERR.puts caller.select{|l| l =~ /rbbt/} * "\n" if @@severity == -1 and not message.empty?
-    #STDERR.puts "#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}[#{severity.to_s}]: " +  message if severity >= @@severity
-    STDERR.puts "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m:#{font_color} " <<  message.strip  << "\033[0m" if severity >= @@severity
+    if severity >= self.severity and not message.empty?
+      str = "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m:#{font_color} " <<  message.strip  << "\033[0m"
+      STDERR.puts str
+      logfile.puts str unless logfile.nil?
+    end
   end
 
   def self.debug(message)
@@ -59,17 +70,17 @@ module Log
 
   case ENV['RBBT_LOG']
   when 'DEBUG' 
-    @@severity = DEBUG
+    self.severity = DEBUG
   when 'LOW' 
-    @@severity = LOW
+    self.severity = LOW
   when 'MEDIUM' 
-    @@severity = MEDIUM
+    self.severity = MEDIUM
   when 'HIGH' 
-    @@severity = HIGH
+    self.severity = HIGH
   when nil
-    @@severity = INFO
+    self.severity = INFO
   else
-    @@severity = ENV['RBBT_LOG'].to_i
+    self.severity = ENV['RBBT_LOG'].to_i
   end
 end
 

data/lib/rbbt/util/misc.rb

@@ -3,10 +3,34 @@ require 'rbbt/util/chain_methods'
 require 'rbbt/resource/path'
 require 'rbbt/annotations'
 require 'net/smtp'
+require 'narray'
 
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  ARRAY_MAX_LENGTH = 10000
+  STRING_MAX_LENGTH = ARRAY_MAX_LENGTH * 10
+  def self.remove_long_items(obj)
+    case
+    when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
+      remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
+    when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
+      remove_long_items(obj.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
+    when (String === obj and obj.length > STRING_MAX_LENGTH)
+      obj[0..STRING_MAX_LENGTH-1] << " TRUNCATED at #{STRING_MAX_LENGTH} (#{obj.length})"
+    when Hash === obj
+      new = {}
+      obj.each do |k,v|
+        new[k] = remove_long_items(v)
+      end
+      new
+    when Array === obj
+      obj.collect do |e| remove_long_items(e) end
+    else
+      obj
+    end
+  end
+
   def self.ensembl_server(organism)
     date = organism.split("/")[1]
     if date.nil?
@@ -192,6 +216,77 @@ end
     "val" =>   "V"
   }
 
+  def self.fast_align(reference, sequence)
+    init_gap = -1
+    gap = -2
+    diff = -2
+    same = 2
+
+    cols = sequence.length + 1
+    rows = reference.length + 1
+
+    a = NArray.int(cols, rows)
+
+
+    for spos in 0..cols-1 do a[spos, 0] = spos * init_gap end
+    for rpos in 0..rows-1 do a[0, rpos] = rpos * init_gap end
+
+    for spos in 1..cols-1 do
+      for rpos in 1..rows-1 do
+        match = a[spos-1,rpos-1] + (sequence[spos-1] != reference[rpos-1] ? diff : same)
+        skip_sequence = a[spos-1,rpos] + gap
+        skip_reference = a[spos,rpos-1] + gap
+        a[spos,rpos] = [match, skip_sequence, skip_reference].max
+      end
+    end
+
+    start = Misc.max(a[-1,0..rows-1])
+    start_pos = a[-1,0..rows-1].to_a.index start
+
+    ref = ''
+    seq = ''
+    rpos = start_pos
+    spos = cols - 1
+
+    while spos > 0 and rpos > 0
+      score = a[spos,rpos]
+      score_match = a[spos-1,rpos-1]
+      score_skip_reference = a[spos,rpos-1]
+      score_skip_sequence = a[spos-1,rpos]
+
+      case
+      when score == score_match + (sequence[spos-1] != reference[rpos-1] ? diff : same)
+        ref << reference[rpos-1]
+        seq << sequence[spos-1]
+        spos -= 1
+        rpos -= 1
+      when score == score_skip_reference + gap
+        ref << reference[rpos-1]
+        seq << '-'
+        rpos -= 1
+      when score == score_skip_sequence + gap
+        seq << sequence[spos-1]
+        ref << '-'
+        spos -= 1
+      else
+        raise "stop"
+      end
+    end
+
+    while (rpos > 0)
+      ref << reference[rpos-1]
+      seq = seq << '-'
+      rpos -= 1    
+    end
+
+    while (spos > 0)
+      seq << sequence[spos-1]
+      ref = ref + '-'
+      spos -= 1
+    end
+
+    [ref.reverse + reference[start_pos..-1], seq.reverse + '-' * (rows - start_pos - 1)]
+  end
   def self.IUPAC_to_base(iupac)
     IUPAC2BASE[iupac]
   end
@@ -366,10 +461,14 @@ end
 
   def self.lock(file, *args)
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
-    lockfile = Lockfile.new(file + '.lock', :max_age => 1200, :suspend => 240)
-    lockfile.lock do
-      yield file, *args
+
+    res = nil
+
+    Lockfile.new(file + '.lock') do
+      res = yield file, *args
     end
+
+    res
   end
 
   def self.common_path(dir, file)
@@ -410,21 +509,23 @@ end
   end
 
   def self.sensiblewrite(path, content)
-    begin
-      case
-      when String === content
-        File.open(path, 'w') do |f|  f.write content  end
-      when (IO === content or StringIO === content)
-        File.open(path, 'w') do |f|  while l = content.gets; f.write l; end  end
-      else
-        File.open(path, 'w') do |f|  end
+    Misc.lock path do
+      begin
+        case
+        when String === content
+          File.open(path, 'w') do |f|  f.write content  end
+        when (IO === content or StringIO === content)
+          File.open(path, 'w') do |f|  while l = content.gets; f.write l; end  end
+        else
+          File.open(path, 'w') do |f|  end
+        end
+      rescue Interrupt
+        FileUtils.rm_f path
+        raise "Interrupted (Ctrl-c)"
+      rescue Exception
+        FileUtils.rm_f path
+        raise $!
       end
-    rescue Interrupt
-      FileUtils.rm_f path
-      raise "Interrupted (Ctrl-c)"
-    rescue Exception
-      FileUtils.rm_f path
-      raise $!
     end
   end
 
@@ -456,12 +557,18 @@ end
       next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
       v = hash[k]
       case
+      when TrueClass === v
+        str << k.to_s << "=>true" 
+      when FalseClass === v
+        str << k.to_s << "=>false" 
       when Hash === v
         str << k.to_s << "=>" << hash2md5(v)
       when Symbol === v
         str << k.to_s << "=>" << v.to_s
       when String === v
-        str << k.to_s << "=>" << v
+        str << k.to_s << "=>" << v[0..10000]
+      when Array === v
+        str << k.to_s << "=>[" << v[0..1000] * "," << "]"
       else
         v_ins = v.inspect
 
@@ -469,7 +576,7 @@ end
         when v_ins =~ /:0x0/
           str << k.to_s << "=>" << v_ins.sub(/:0x[a-f0-9]+@/,'')
         else
-          str << k.to_s << "=>" << v.to_s
+          str << k.to_s << "=>" << v_ins
         end
 
       end

data/lib/rbbt/util/open.rb

@@ -104,9 +104,7 @@ module Open
   
   def self.add_cache(url, data, options = {})
     file = File.join(REMOTE_CACHEDIR, digest_url(url, options))
-    Misc.lock(file) do
-      Misc.sensiblewrite(file, data)
-    end
+    Misc.sensiblewrite(file, data)
   end
 
   # Grep

data/lib/rbbt/workflow.rb

@@ -107,7 +107,7 @@ module Workflow
       end
     end
 
-    raise "Workflow not found: #{ wf_name }"
+    raise "Workflow not found our could not be loaded: #{ wf_name }"
   end
 
   def self.require_workflow(wf_name)

data/lib/rbbt/workflow/accessor.rb

@@ -19,7 +19,14 @@ class Step
 
   def info
     return {} if not File.exists? info_file
-    YAML.load(Open.open(info_file)) || {}
+    begin
+      File.open(info_file) do |file|
+        YAML.load(file) || {}
+      end
+    rescue
+      Log.debug "Error loading yaml: " + Open.read(info_file)
+      raise $!
+    end
   end
 
   def set_info(key, value)
@@ -44,19 +51,23 @@ class Step
   end
 
   def message(message)
-    set_info(:messages, messages << message)
+    set_info(:messages, (messages || []) << message)
   end
 
   def log(status, message = nil)
     if message
-      Log.low "#{ status }: #{ message }"
+      Log.low "[#{ status }] #{ message }: #{path}"
     else
-      Log.low "#{ status }"
+      Log.low "[#{ status }]: #{path}"
     end
     self.status = status
     message(message) unless message.nil?
   end
 
+  def started?
+    File.exists? info_file
+  end
+
   def done?
     status = info[:status]
     status == :done or status == :error

data/lib/rbbt/workflow/step.rb

@@ -4,14 +4,15 @@ require 'rbbt/util/log'
 require 'rbbt/workflow/accessor'
 
 class Step
-  attr_accessor :path, :task, :inputs, :dependencies
+  attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
 
   class Aborted < Exception; end
 
-  def initialize(path, task = nil, inputs = nil, dependencies = nil)
+  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
     @path = path
     @task = task
+    @bindings = bindings
     @dependencies = case
                     when dependencies.nil? 
                       []
@@ -30,14 +31,13 @@ class Step
   end
 
   def exec
-    result = @task.exec_in self, *@inputs
+    result = @task.exec_in((bindings ? bindings : self), *@inputs)
     prepare_result result, @task.result_description
   end
 
   def join
     if @pid.nil?
       while not done? do
-        Log.debug "Waiting: #{info[:step]}"
         sleep 5
       end
     else
@@ -50,12 +50,22 @@ class Step
 
   def run(no_load = false)
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
-      log task.name, "Starting task: #{ name }"
+      FileUtils.rm info_file if File.exists? info_file
+      log((task.name || "unnamed task"), "Starting task")
       set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
-      @dependencies.each{|dependency| dependency.run true}
-      set_info :status, :start
-      set_info :inputs, Misc.zip2hash(task.inputs, @inputs) unless task.inputs.nil?
-      res = exec
+      @dependencies.each{|dependency| 
+        log dependency.task.name || "dependency", "Processing dependency: #{ dependency.path }"
+        dependency.run true
+      }
+      set_info :status, :started
+      set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
+      res = begin
+              exec
+            rescue Exception
+              set_info :backtrace, $!.backtrace
+              log(:error, "#{$!.class}: #{$!.message}")
+              raise $!
+            end
       set_info :status, :done
       res
     end
@@ -66,11 +76,12 @@ class Step
   def fork
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
     @pid = Process.fork do
+      trap(:INT) { raise Step::Aborted.new "INT signal recieved" }
+      FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
       begin
-        trap(:INT) { raise Step::Aborted.new "INT signal recieved" }
         run
-      rescue Exception
-        log(:error, "#{$!.class}: #{$!.message}")
+      rescue
+        exit -1
       end
     end
     set_info :pid, @pid

data/test/rbbt/resource/test_path.rb

@@ -5,6 +5,18 @@ require 'test/unit'
 
 class TestTSV < Test::Unit::TestCase
 
+  def test_find
+    name = 'test_name_for_unexistent_file'
+    path = Path.setup "tmp/#{ name }"
+
+    TmpFile.with_file do |dir|
+      assert File.join(ENV['HOME'], path), path.find(nil, nil, {:root => File.join(dir, '{TOPLEVEL}/{SUBPATH}'), :default => :user, :user => File.join(ENV['HOME'], "{TOPLEVEL}", "{SUBPATH}")})
+      FileUtils.mkdir_p File.dirname(File.join(dir, path))
+      FileUtils.touch File.join(dir, path)
+      assert File.join(dir, "tmp/test"), path.find(nil, nil, {:root => File.join(dir, '{TOPLEVEL}/{SUBPATH}'), :default => :user, :user => File.join(ENV['HOME'], "{TOPLEVEL}", "{SUBPATH}")})
+    end
+  end
+
   def test_prev
     path = Path.setup "/tmp"
     assert_equal "/tmp/bar/foo", path.foo("bar")

data/test/rbbt/test_persist.rb

@@ -8,6 +8,7 @@ module TestAnnotation
 
   self.annotation :test_annotation
 end
+
 class TestPersist < Test::Unit::TestCase
 
   def test_annotation_persist
@@ -40,6 +41,51 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_bdb
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :double, TokyoCabinet::BDB)
+      repo["test:string1"] = [["STR1"]]
+      repo["test:string2"] = [["STR2"]]
+      repo["other_test:string3"] = [["STR2"]]
+
+      assert_equal ["test:string1", "test:string2"].sort, repo.range("test:" << 0.chr, false, "test:" << 255.chr, false).sort
+      assert_equal ["other_test:string3"].sort, repo.range("other_test:" << 0.chr, false, "other_test:" << 255.chr, false).sort
+    end
+  end
+
+  def test_annotation_persist_repo
+    TmpFile.with_file do |tmp|
+      repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
+
+      entity1 = "Entity 1"
+      entity2 = "Entity 2"
+
+      TestAnnotation.setup(entity1, :test_annotation => "1")
+      TestAnnotation.setup(entity2, :test_annotation => "2")
+
+      annotations = [entity1, entity2]
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.first
+      assert_equal "Entity 2", persisted_annotations.last
+      assert_equal "1", persisted_annotations.first.test_annotation
+      assert_equal "2", persisted_annotations.last.test_annotation
+
+      persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
+        annotations
+      end
+
+      assert_equal "Entity 1", persisted_annotations.sort.first
+      assert_equal "Entity 2", persisted_annotations.sort.last
+      assert_equal "1", persisted_annotations.sort.first.test_annotation
+      assert_equal "2", persisted_annotations.sort.last.test_annotation
+    end
+  end
+
+
   def test_array_persist
     TmpFile.with_file do |tmp|
       10.times do

data/test/rbbt/tsv/test_index.rb

@@ -103,7 +103,7 @@ row3    A    a|B    Id4
       assert_equal "OtherID", index.fields.first
 
       index = tsv.index(:order => true, :persist => false)
-      assert_equal "Id3", index['a'].first
+      assert_equal "Id1", index['a'].first
     end
   end
 

data/test/rbbt/util/test_misc.rb

@@ -96,7 +96,6 @@ class TestMisc < Test::Unit::TestCase
     a = [[1],[2]]
     a = NamedArray.setup a, %w(1 2)
     a.merge [3,4]
-    ddd a
     assert_equal [1,3], a[0]
   end
 
@@ -151,6 +150,18 @@ class TestMisc < Test::Unit::TestCase
     assert_equal Math.sqrt(2), Misc.sd([1,3])
   end
 
+  def test_align_small
+    reference = "AABCDEBD"
+    sequence  = "ABCD"
+    assert_equal '-ABCD---', Misc.fast_align(reference, sequence).last
+  end
+
+  def test_align_real
+    reference = "SGNECNKAIDGNKDTFWHTFYGANGDPKPPPHTYTIDMKTTQNVNGLSMLPRQDGNQNGWIGRHEVYLSSDGTNW"
+    sequence  = "TYTIDMKTTQNVNGLSML"
+    assert_equal "--------------------------------TYTIDMKTTQNVNGLSML-------------------------", Misc.fast_align(reference, sequence).last
+  end
+
 #  def test_divide
 #    assert_equal 2, Misc.divide(%w(1 2 3 4 5 6 7 8 9),2).length
 #  end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-util
 version: !ruby/object:Gem::Version 
-  hash: 55
+  hash: 51
   prerelease: 
   segments: 
   - 4
-  - 2
+  - 3
   - 0
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-13 00:00:00 +01:00
-default_executable: 
+date: 2012-01-31 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
@@ -116,6 +115,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: narray
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id008
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez@cnio.es
 executables: 
@@ -209,7 +222,6 @@ files:
 - bin/rbbt_query.rb
 - bin/rbbt_exec.rb
 - bin/rbbt_Rutil.rb
-has_rdoc: true
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []
 
@@ -239,7 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.8.12
 signing_key: 
 specification_version: 3
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)