rbbt-util 5.14.28 → 5.14.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46848d2c59e5582f44b698be77f427d886682c4f
-  data.tar.gz: c3ae39b55c1f9d44975679aa2916cdcb198cd7e1
+  metadata.gz: 1ed65d89a02680821214ab3141ef167565045291
+  data.tar.gz: 95402199abefb0228dc7788731a1c2b0487c9bf6
 SHA512:
-  metadata.gz: 6fe50e3bb01ed9b718f71a19f29b5697af0aacbc9ce602b93f802ba1400074491782f32774cb56d832c50befd99386566732e8a9941ba2500957e2f66ef1d9b5
-  data.tar.gz: 600ac43a4bd563779943a3061a151d3dacb5b3d2cfa5ff98d7f90f600bc29578bd34c3161af06d8d97ec7d32f1c9896f21518ff0cd2837f00f569d2348894d6c
+  metadata.gz: 8643b977a908ea2447d070aef682d4f5bafc3360f380cbc3b43be992ba6bad689cfba5ee2ed546fce64672f133fb5f2676b13e4088d6a60f0c91ba2bef741108
+  data.tar.gz: 25b4bf073c68ee563eb17d77b00af6a38f9d25d10520f834167bb94e6b8daea866c048264f97a9b6afc2917d7d32b16c4b41a52aa123176541b3e84f8bd83b3b

data/lib/rbbt/association/index.rb

@@ -70,7 +70,7 @@ module Association
       source_type = Entity.formats[source_field] 
       target_type = Entity.formats[target_field]
 
-      source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s]  
+      source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] 
       target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s]
 
       [source_entities, target_entities]
@@ -89,7 +89,11 @@ module Association
       end
 
       matches = source.uniq.inject([]){|acc,e| 
-        acc.concat(match(e)) 
+        if block_given?
+          acc.concat(match(e))
+        else
+          acc.concat(match(e))
+        end
       }
 
       return matches if target == :all or target == "all"

data/lib/rbbt/association.rb

@@ -133,12 +133,13 @@ module Association
     target_field = tsv.fields.first
 
     if source_final_format and source_field != source_final_format and
-      Entity.formats[source_field] and
-      Entity.formats[source_final_format].all_formats.include? source_field
+      Entity.formats[source_field] 
+
       Log.debug("Changing source format from #{tsv.key_field} to #{source_final_format}")
 
       tsv.with_unnamed do
-        tsv = tsv.change_key source_final_format, :identifiers => Organism.identifiers(tsv.namespace), :persist => true
+        identifiers = tsv.identifiers || Organism.identifiers(tsv.namespace)
+        tsv = tsv.change_key source_final_format, :identifiers => identifiers, :persist => true
       end
     end
 
@@ -221,7 +222,14 @@ module Association
 
     case file
     when Proc
-      return load_tsv(file.call, options)
+      res = file.call
+      tsv = case res
+            when TSV, Path
+              return load_tsv(res, options)
+            else
+              tsv = TSV.open(res, :unnamed => true)
+              return load_tsv(tsv, options)
+            end 
     when TSV
       key_field, *fields = all_fields = file.all_fields
     else 
@@ -257,6 +265,7 @@ module Association
       tsv = load_tsv(file, options)
 
       tsv.annotate(data)
+
       data.serializer = tsv.type 
 
       tsv.with_unnamed do

data/lib/rbbt/knowledge_base.rb

@@ -358,7 +358,7 @@ class KnowledgeBase
     end
   end
 
-  def subset(name, entities)
+  def subset(name, entities, &block)
     entities = case entities
                when :all
                  {:target => :all, :source => :all}
@@ -374,18 +374,20 @@ class KnowledgeBase
 
     repo = get_index name
 
-    begin
-      s = repo.subset_entities(entities)
-      setup(name, s)
-    rescue Exception
-      target = entities[:target]
-      source = entities[:source]
-      if target or source
-        entities[:target] = source 
-        entities[:source] = target
-      end
-      setup(name, repo.reverse.subset_entities(entities), true)
-    end
+    matches = begin
+                s = repo.subset_entities(entities, &block)
+                setup(name, s)
+              rescue Exception
+                target = entities[:target]
+                source = entities[:source]
+                if target or source
+                  entities[:target] = source 
+                  entities[:source] = target
+                end
+                setup(name, repo.reverse.subset_entities(entities, &block), true)
+              end
+
+    block_given? ? matches.select(&block) : matches
   end
 
   def translate(entities, type)

data/lib/rbbt/persist.rb

@@ -218,7 +218,7 @@ module Persist
   end
 
   def self.get_result(path, type, persist_options, lockfile, &block)
-    res = yield
+    res = yield path
     stream = res if IO === res
     stream = res.stream if res.respond_to? :stream
 
@@ -317,6 +317,7 @@ module Persist
     if FalseClass === persist_options[:persist]
       yield
     else
+      persist_options[:update] ||= true if persist_options[:persist].to_s == "update"
       other_options = Misc.process_options persist_options, :other
       path = persistence_path(name, persist_options, other_options || {})
 

data/lib/rbbt/resource/rake.rb

@@ -47,8 +47,9 @@ module Rake
           Rake::Task.clear
           Rake::FileTask.clear_files
         end
-      rescue
+      rescue Exception
         Log.error "Error in rake: #{$!.message}"
+        Log.exception $!
         Kernel.exit! -1
       end
     }

data/lib/rbbt/resource.rb

@@ -122,7 +122,12 @@ module Resource
       raise "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
     end
 
-    final_path = path.respond_to?(:find) ? (force ? path.find(:default) : path.find) : path
+    if path.respond_to?(:find) 
+      final_path = force ? path.find(:default) : path.find
+    else
+      final_path = path
+    end
+
     if not File.exists? final_path or force
       Log.medium "Producing: #{ final_path }"
       lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})

data/lib/rbbt/tsv/accessor.rb

@@ -9,7 +9,7 @@ module TSV
   attr_accessor :unnamed, :serializer_module, :entity_options, :entity_templates
 
   def info
-    {:key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed}.delete_if{|k,v| v.nil? }
+    {:key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed, :cast => cast}.delete_if{|k,v| v.nil? }
   end
 
   def annotate(tsv)

data/lib/rbbt/tsv/attach.rb

@@ -182,7 +182,7 @@ module TSV
     unless TSV === other
       other_identifier_file = other.identifier_files.first if other.respond_to? :identifier_files
       other = TSV.open(other, :persist => options[:persist_input] == true)
-      other.identifiers = other_identifier_file
+      other.identifiers ||= other_identifier_file
     end
 
     fields = other.fields - [key_field].concat(self.fields) if fields.nil?  or fields == :all 

data/lib/rbbt/tsv/dumper.rb

@@ -45,6 +45,7 @@ module TSV
       str = TSV.header_lines(key_field, fields, options)
 
       Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
+
       @in_stream.puts str
     end
 

data/lib/rbbt/tsv/manipulate.rb

@@ -631,10 +631,11 @@ module TSV
     self
   end
 
-  def transpose(key_field)
+  def transpose(key_field="Unkown ID")
     raise "Transposing only works for TSVs of type :list" unless type == :list
     new_fields = keys
-    new = TSV.setup({}, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
+    new = self.annotate({})
+    TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
 
     through do |key, values|
       fields.zip(values) do |new_key, value|
@@ -643,10 +644,6 @@ module TSV
       end
     end
 
-    new.entity_options = entity_options
-    new.entity_templates = entity_templates
-    new.namespace = namespace
-
     new
   end
 end

data/lib/rbbt/tsv/matrix.rb

@@ -0,0 +1,20 @@
+module TSV
+  def self.read_matrix(tsv, field_format = "ID", value_format = "Value")
+    tsv = TSV.open(tsv) unless TSV === tsv
+
+    key_field, *fields = tsv.all_fields
+    options = tsv.options.merge(:key_field => key_field, :fields => [field_format, value_format], :type => :double, :cast => nil)
+
+    options[:filename] ||= tsv.filename
+    options[:identifiers] ||= tsv.identifier_files.first
+
+    dumper = TSV::Dumper.new(options)
+
+    dumper.init
+    TSV.traverse tsv, :into => dumper do |key, values|
+      [key, [fields, values]]
+    end
+
+    dumper.stream
+  end
+end

data/lib/rbbt/tsv/parser.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/cmd'
 module TSV
   class Parser
-    attr_accessor :stream, :filename, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream, :preamble
+    attr_accessor :stream, :filename, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream, :preamble, :identifiers
 
     class SKIP_LINE < Exception; end
     class END_PARSING < Exception; end
@@ -392,6 +392,8 @@ module TSV
       @type ||= Misc.process_options(options, :type) || :double
       @type ||= :double
 
+      @identifiers = Misc.process_options(options, :identifiers) 
+
       @filename = Misc.process_options(options, :filename) 
       @filename ||= stream.filename if stream.respond_to? :filename
 
@@ -472,6 +474,7 @@ module TSV
       data.fields = @fields
       data.namespace = @namespace
       data.filename = @filename
+      data.identifiers = @identifiers
       data.cast = @cast if Symbol === @cast
       data
     end
@@ -559,6 +562,8 @@ module TSV
               break
             end
           rescue END_PARSING
+            stream.close unless stream.closed?
+            begin stream.join; rescue Exception; end if stream.respond_to? :join and not stream.joined?
             break
           rescue Errno::EPIPE
             Log.error "Pipe closed while parsing #{Misc.fingerprint stream}: #{$!.message}"
@@ -572,7 +577,7 @@ module TSV
         end
       ensure
         stream.close unless stream.closed?
-        stream.join if stream.respond_to? :join
+        stream.join if stream.respond_to? :join and not stream.joined?
       end
 
       self

data/lib/rbbt/tsv/util.rb

@@ -167,7 +167,7 @@ module TSV
     str = "" 
     str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
-      str << "#" << key_field << sep << fields * sep << "\n"
+      str << "#" << (key_field || "ID") << sep << fields * sep << "\n"
     end
 
     str

data/lib/rbbt/tsv.rb

@@ -17,6 +17,7 @@ require 'rbbt/tsv/field_index'
 require 'rbbt/tsv/parallel'
 require 'rbbt/tsv/stream'
 require 'rbbt/tsv/melt'
+require 'rbbt/tsv/matrix'
 
 module TSV
   class << self

data/lib/rbbt/util/R.rb

@@ -26,7 +26,7 @@ source('#{UTIL}');
     Log.debug{"R Script:\n#{ cmd }"}
 
     if options.delete :monitor
-      io = CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
+      io = CMD.cmd('R --vanilla', options.merge(:in => cmd, :pipe => true, :log => true))
       while line = io.gets
         puts line
       end

data/lib/rbbt/util/cmd.rb

@@ -104,7 +104,8 @@ module CMD
     pipe       = options.delete(:pipe)
     post       = options.delete(:post)
     log        = options.delete(:log)
-    dont_close_in        = options.delete(:dont_close_in)
+    no_fail    = options.delete(:no_fail)
+    dont_close_in  = options.delete(:dont_close_in)
 
     log = true if log.nil?
 
@@ -205,7 +206,7 @@ module CMD
       end
 
       #SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
-      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
+      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true, :no_fail => no_fail
 
       sout
     else
@@ -217,7 +218,7 @@ module CMD
         serr.close
       end
 
-      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
+      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true, :no_fail => no_fail
       out = StringIO.new sout.read
       sout.close unless sout.closed?
 

data/lib/rbbt/util/log.rb

@@ -142,7 +142,7 @@ module Log
     message ||= block.call if block_given?
     return if message.nil?
 
-    time = Time.now.strftime("%m/%d/%y-%H:%M:%S")
+    time = Time.now.strftime("%m/%d/%y-%H:%M:%S.%L")
 
     sev_str = severity.to_s
 

data/lib/rbbt/util/misc/bgzf.rb

@@ -38,7 +38,12 @@ module Bgzf
                     pos = 0
                     while true do
                       blockdata_offset = tell
-                      block = read_block
+                      block = begin
+                                read_block
+                              rescue Exception
+                                raise "BGZF seems to be buggy so some compressed files will not decompress right. Try uncompressing #{filename}" if $!.message =~ /BGFZ.*expected/
+                                raise $!
+                              end
                       break unless block
                       index << [pos, blockdata_offset]
                       pos += block.length

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -1,8 +1,8 @@
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail
 
   def self.setup(stream, options = {}, &block)
-    threads, pids, callback, filename, autojoin, lockfile = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin, :lockfile
+    threads, pids, callback, filename, autojoin, lockfile, no_fail = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin, :lockfile, :no_fail
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -10,6 +10,7 @@ module ConcurrentStream
     stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin
+    stream.no_fail = no_fail
 
     callback = block if block_given?
     if stream.callback and callback
@@ -60,7 +61,7 @@ module ConcurrentStream
       @pids.each do |pid| 
         begin
           Process.waitpid(pid, Process::WUNTRACED)
-          raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success?
+          raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
       end 
@@ -76,12 +77,13 @@ module ConcurrentStream
   end
 
   def join
+    @joined = true
+
     join_threads
     join_pids
 
     join_callback
 
-    @joined = true
     lockfile.unlock if lockfile and lockfile.locked?
     close unless closed?
   end

data/lib/rbbt/util/open.rb

@@ -132,7 +132,7 @@ module Open
         CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
-      CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
+      CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
     end
   end
 

data/lib/rbbt/workflow.rb

@@ -216,6 +216,10 @@ module Workflow
   def exec_exports
     @exec_exports ||= []
   end
+  
+  def all_exports
+    @all_exports ||= asynchronous_exports + synchronous_exports + exec_exports
+  end
 
   # {{{ JOB MANAGEMENT
   DEFAULT_NAME="Default"

data/share/rbbt_commands/check_bgzf

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Check the integrity of a BGZF file
+
+$ rbbt check_bgzf <filename.bgz>
+
+-h--help Print this help
+
+EOF
+
+rbbt_usage and exit 0 if defined? rbbt_usage and (options[:help] or ARGV.empty?)
+
+file = ARGV.shift
+
+begin
+  Open.read(file)
+rescue Exception
+  puts Log.color :red, "#{ file } is Broken"
+  exit -1
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.28
+  version: 5.14.29
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-16 00:00:00.000000000 Z
+date: 2014-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -176,6 +176,7 @@ files:
 - lib/rbbt/tsv/filter.rb
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
+- lib/rbbt/tsv/matrix.rb
 - lib/rbbt/tsv/melt.rb
 - lib/rbbt/tsv/parallel.rb
 - lib/rbbt/tsv/parallel/through.rb
@@ -254,6 +255,7 @@ files:
 - share/rbbt_commands/benchmark/pthrough
 - share/rbbt_commands/benchmark/throughput
 - share/rbbt_commands/benchmark/tsv
+- share/rbbt_commands/check_bgzf
 - share/rbbt_commands/color
 - share/rbbt_commands/conf/web_user/add
 - share/rbbt_commands/conf/web_user/list