rbbt-util 5.41.1 → 5.43.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bd731f2fb8604116f9baf947e8e0b5f5e8bd1b252773864f5a76e9c4b52126d
-  data.tar.gz: 82c364cc76d5921e49197c410d05a919e8b71a96d0499827b0546bf402c401ae
+  metadata.gz: d1bbf2cefca44eecad7465a8d4dc1b88f53f18d1e63e310353061b43ad46bed9
+  data.tar.gz: f54f659da9eca312d116430d88019f067ce5ccf519724a2a448e3b4d31dfb767
 SHA512:
-  metadata.gz: 655e499e17ea4806f278ecaf83521c200aa978ace61c27f5aac0490bc99a40cac42498f323155ee3c547a2a98d23271a8fa6d3b424180c87b249b50219fb7235
-  data.tar.gz: c3ef8cc5790db9bb5df6d0853542355b267e2f8f26c4b6bc8305729b793eff8832a33479a53339605bcaf8b450a500dc58a9bd5557e1d993f8c8b72228ab1793
+  metadata.gz: eaad34515129ffd123064e9af920b5f83f3897c957de45e3bef04a1101d961d419863729f21665773b81f535f3fdba45d490ce41870f9db87f0e0e5d5f987233
+  data.tar.gz: d48552f30e6434b0e27e173eb62f41a3126ecb9065cba58f2f5737f8e3aef0e2183503076a40cc6586832d90458ae5e572a55171758452f3699a4225704bc25e

data/lib/rbbt/annotations/util.rb

@@ -43,7 +43,7 @@ module Annotated
   end
 
   def self.resolve_array(entry)
-    if entry =~ /^Array:/
+    if String === entry && entry =~ /^Array:/
       entry["Array:".length..-1].split("|")
     else
       entry

data/lib/rbbt/association/database.rb

@@ -93,7 +93,7 @@ module Association
       tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
 
       tsv.with_monitor(options[:monitor]) do
-        tsv = tsv.reorder source_field, fields, :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
+        tsv = tsv.reorder source_field, tsv.all_fields.values_at(*field_pos), :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
       end
 
       tsv.key_field = source_header
@@ -216,6 +216,7 @@ module Association
                  open_stream(stream, options.dup)
                end
 
+    database.filename = file if Path === file && file.identifier_files.any?
     database.entity_options = options[:entity_options] if options[:entity_options]
 
     database

data/lib/rbbt/association/index.rb

@@ -38,6 +38,7 @@ module Association
       data.fields = fields[1..-1]
       data.type = :list
       data.serializer ||= serializer
+      data.filename ||= file if String === file
 
       database.with_unnamed do
         database.with_monitor(options[:monitor]) do

data/lib/rbbt/association/util.rb

@@ -115,8 +115,14 @@ module Association
     source_field = specs[:source][0]
     target_field = specs[:target][0]
 
-    source_pos = all_fields.index source_field
-    target_pos = all_fields.index target_field
+    #source_pos = all_fields.index source_field
+    #target_pos = all_fields.index target_field
+   
+    source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
+    target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
+
+    source_pos = source_pos == :key ? 0 : source_pos + 1
+    target_pos = target_pos == :key ? 0 : target_pos + 1
 
     source_header = specs[:source][1] || specs[:source][0]
     target_header = specs[:target][1] || specs[:target][0]
@@ -140,7 +146,11 @@ module Association
       field_headers << header
     end
 
-    field_pos = info_fields.collect{|f| raise "Field #{f} not found. Options: #{all_fields * ", "}" unless all_fields.include?(f); f == :key ? 0 : all_fields.index(f);  }
+    field_pos = info_fields.collect do |f| 
+      p = TSV.identify_field all_fields.first, all_fields[1..-1], f
+      p == :key ? 0 : p + 1
+    end
+
     field_pos.delete source_pos
 
     source_format = specs[:source][2]

data/lib/rbbt/entity.rb

@@ -289,16 +289,17 @@ module Entity
       end
 
       def self.unpersist(method_name)
-        return unless persisted? method_name
-        orig_name = UNPERSISTED_PREFIX + method_name.to_s
-
+        return unless orig_name = persisted?(method_name)
         alias_method method_name, orig_name
         remove_method orig_name
       end
 
       def self.persisted?(method_name)
-        orig_name = UNPERSISTED_PREFIX + method_name.to_s
-        instance_methods.include? orig_name.to_sym
+        ["", "_multiple_", "_single_"].each do |type|
+          orig_name = (UNPERSISTED_PREFIX + type + method_name.to_s).to_sym
+          return orig_name if instance_methods.include?(orig_name)
+        end
+        return false
       end
 
       def self.with_persisted(method_name)

data/lib/rbbt/hpc/batch.rb

@@ -15,6 +15,7 @@ module HPC
     when 'pbs'
       HPC::PBS
     when 'auto'
+      $previous_commands = [] if $previous_commands.nil?
       case $previous_commands.last
       when 'slurm'
         HPC::SLURM
@@ -735,7 +736,7 @@ env > #{batch_options[:fenv]}
     end
 
     def job_queued(job)
-      job_status(job).split(/\s+/).include?(job.to_s)
+      job_status(job).split(/[\s\.]+/).include?(job.to_s)
     end
 
     def wait_for_job(batch_dir, time = 1)

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -152,7 +152,7 @@ module HPC
     end
 
     def self.job_batches(rules, job)
-      job_chains = self.job_chains(rules, job)
+      job_chains = self.job_chains(rules, job).dup
 
       workload = job_workload(job).uniq
 

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -39,9 +39,8 @@ module HPC
       (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
     end
 
-    def self.job_chains(rules, job)
-      @@job_chains ||= {}
-      @@job_chains[Misc.digest([rules, job.path].inspect)] ||= 
+    def self.job_chains(rules, job, computed = {})
+      computed[Misc.fingerprint([rules, job.path, job.object_id])] ||=
         begin
           chains = self.parse_chains(rules)
 
@@ -55,7 +54,7 @@ module HPC
             dep_matches = check_chains(chains, dep)
             common = matches & dep_matches
 
-            dep_chains = job_chains(rules, dep)
+            dep_chains = job_chains(rules, dep, computed)
             found = []
             dep_chains.each do |match,info|
               if common.include?(match)

data/lib/rbbt/knowledge_base/traverse.rb

@@ -146,7 +146,7 @@ class KnowledgeBase
       if conditions
         Misc.tokenize(conditions).each do |condition|
           if condition.index "="
-            key, value = conditions.split("=")
+            key, value = condition.split("=")
             matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
           else
             matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
@@ -158,17 +158,19 @@ class KnowledgeBase
     end
 
     def id_dbs(db)
+      # ToDo: Revise this, I'm not sure what id does anymore
+      # I think it deals with syndication
       if db.include? '?'
-        all_dbs = kb.registry.keys
+        all_dbs = kb.registry.keys.collect{|k| k.to_s }
         _name, _sep, _kb = db.partition("@")
         case
         when _name[0] == '?'
           dbs = all_dbs.select{|_db| 
             n,_s,d=_db.partition("@"); 
-            d.nil? or d.empty? or (d == _kd and assignments[_name].include?(n))
+            d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
           }
         when _kb[0] == '?'
-          dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d)}
+          dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
         end
       else
         dbs = [db]
@@ -181,11 +183,12 @@ class KnowledgeBase
       all_matches = []
       path_rules = []
       acc_var = nil
+      pre_acc_var_assignments = nil
       rules.each do |rule|
         rule = rule.strip
         next if rule.empty?
 
-        if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/)
+        if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
           Log.debug "Traverse rule: #{rule}"
           path_rules << rule
 
@@ -199,17 +202,19 @@ class KnowledgeBase
 
             next if matches.nil? or matches.empty?
 
-            if db.include? '?'
-              _name, _sep, _kb = db.partition("@")
-              case
-              when _kb[0] == '?'
-                assignments[_kb] ||= []
-                assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
-              when _name[0] == '?'
-                assignments[_name] ||= []
-                assignments[_name] << _db.partition("@").first
-              end
-            end
+            # ToDo: Revise this, I'm not sure what id does anymore
+            #
+            #if db.include? '?'
+            #  _name, _sep, _kb = db.partition("@")
+            #  case
+            #  when _kb[0] == '?'
+            #    assignments[_kb] ||= []
+            #    assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
+            #  when _name[0] == '?'
+            #    assignments[_name] ||= []
+            #    assignments[_name] << _db.partition("@").first
+            #  end
+            #end
 
             matches.each do |m|
               rule_matches << m
@@ -244,11 +249,14 @@ class KnowledgeBase
 
         elsif m = rule.match(/(\?[^\s{]+)\s*{/)
           acc_var = m.captures.first
+          pre_acc_var_assignments = assignments.dup
           Log.debug "Start assign block: #{acc_var}"
         elsif m = rule.match(/^\s*}\s*$/)
           Log.debug "Close assign block: #{acc_var}"
           saved_assign = assignments[acc_var]
           assignments.clear
+          assignments.merge!(pre_acc_var_assignments)
+          pre_acc_var_assignments = nil
           assignments[acc_var] = saved_assign
           all_matches = []
           path_rules = []

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -21,7 +21,7 @@ module Persist
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-      database = CONNECTIONS[path] ||= tokyocabinet_class.new
+      database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
 
       if big and not Open.exists?(path)
         database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 

data/lib/rbbt/resource/path.rb

@@ -161,8 +161,13 @@ module Path
   def find(where = nil, caller_lib = nil, paths = nil)
 
     if located?
-      self.original ||= self
-      return self
+      path = self
+      path = path + '.gz' if File.exist?(path + '.gz')
+      path = path + '.bgz' if File.exist?(path + '.bgz')
+      self.annotate(path)
+
+      path.original = self
+      return path
     end
 
     if where == :all || where == 'all'

data/lib/rbbt/resource.rb

@@ -374,7 +374,7 @@ url='#{url}'
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
             unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
             unlocated.gsub!(/\/+/,'/')
-            if self.subdir && ! self.subdir.empty?
+            if self.subdir && ! self.subdir.empty? && unlocated.include?(subdir)
               subdir = self.subdir
               subdir += "/" unless subdir.end_with?("/")
               unlocated[subdir] = ""

data/lib/rbbt/tsv/change_id.rb

@@ -8,8 +8,8 @@ module TSV
 
     identifiers, persist_input = Misc.process_options options, :identifiers, :persist_input
 
-    identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? and tsv.namespace
-
+    identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? && tsv.namespace && 
+      defined?(Organism) && Organism.identifiers(tsv.namespace).exists?
 
     if ! tsv.fields.include?(format)
       new = {}
@@ -58,7 +58,10 @@ module TSV
 
     identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
     identifiers = tsv.identifier_files.first if identifiers.nil?
-    identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
+
+    identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? && tsv.namespace && 
+      defined?(Organism) && Organism.identifiers(tsv.namespace).exists?
+
     identifiers.namespace ||= tsv.namespace
 
     fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -181,6 +181,48 @@ module TSV
     end
   end
 
+  def self.traverse_enumerable(enum, options = {}, &block)
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
+
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        while enum.any?
+          e = enum.pop
+          begin
+            callback.call yield(e)
+          rescue Exception
+            Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
+            raise $!
+          ensure
+            bar.tick if bar
+          end
+        end
+      else
+        bar.init if bar
+        while enum.any?
+          e = enum.pop
+          begin
+            yield e
+          rescue Exception
+            Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
+            raise $!
+          ensure
+            bar.tick if bar
+          end
+        end
+      end
+
+    rescue
+      error = true
+      raise $!
+    ensure
+      join.call(error) if join
+      Log::ProgressBar.remove_bar(bar, error) if bar
+    end
+  end
+
   def self.traverse_priority_queue(queue, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
@@ -396,6 +438,8 @@ module TSV
         else
           raise "Can not open obj for traversal #{Misc.fingerprint obj}"
         end
+      when Enumerable
+        traverse_enumerable(obj, options, &block)
       when nil
         raise "Can not traverse nil object into #{stream_name(options[:into])}"
       else

data/lib/rbbt/tsv/parser.rb

@@ -281,19 +281,20 @@ module TSV
       end
 
       values = values.collect{|v| v.length != num ? [v.first] * num : v}
+
       all = values
       all.unshift keys
-      Misc.zip_fields(all).each do |values|
-        key = values.shift
+      Misc.zip_fields(all).each do |vs|
+        key = vs.shift
         if data.include? key
-          data[key] = data[key].zip(values).collect do |old, new|
-            old.push new
-            old
+          data[key] = data[key].zip(vs).collect do |old, new|
+            old + [new]
           end
         else
-          data[key] = values.collect{|v| [v]}
+          data[key] = vs.collect{|v| [v] }
         end
       end
+
       nil
     end
 

data/lib/rbbt/tsv/serializers.rb

@@ -63,7 +63,7 @@ module TSV
   class StringDoubleArraySerializer
     def self.dump(array)
       begin
-      array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
+        array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
       rescue Encoding::CompatibilityError
         array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
       end

data/lib/rbbt/tsv/util.rb

@@ -168,6 +168,8 @@ module TSV
       end
     when Set
       get_stream(file.to_a, open_options)
+    when Enumerable
+      file
     else
       raise "Cannot get stream from: #{file.inspect}"
     end
@@ -189,6 +191,16 @@ module TSV
       pos = fields.index field
       return pos if pos
       return identify_field(key_field, fields, field.to_i) if field =~ /^\d+$/
+      if fields.select{|f| f.include?("(") }.any?
+        simplify_fields = fields.collect do |f|
+          if m = f.match(/(.*)\s+\(.*\)/)
+            m[1]
+          else
+            f
+          end
+        end
+        return identify_field(key_field, simplify_fields, field)
+      end
       raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}" if pos.nil?
     else
       raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}"

data/lib/rbbt/util/R.rb

@@ -50,7 +50,7 @@ source('#{UTIL}');
           Log.debug "R: " <<  line
         end
       end
-      nil
+      io.join if io.respond_to? :join
     else
       CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd, :xvfb => options[:xvfb]))
     end

data/lib/rbbt/util/config.rb

@@ -18,7 +18,7 @@ module Rbbt::Config
     Log.debug "Loading config file: #{ file }"
     TSV.traverse file, :type => :array do |line|
       next if line =~ /^#/
-      key, value, *tokens = line.strip.split(/\s/)
+      key, value, *tokens = line.strip.split(/\s+/)
 
       self.add_entry(key, value, tokens) if key
     end

data/lib/rbbt/util/log/progress.rb

@@ -9,7 +9,7 @@ module Log
 
   def self.no_bar
     @@no_bar = false unless defined?(@@no_bar)
-    @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
+    (@@no_bar || ENV["RBBT_NO_PROGRESS"]).to_s == "true"
   end
 
   class ProgressBar

data/lib/rbbt/util/log.rb

@@ -399,6 +399,10 @@ module Log
         end
       end
     end
+    LOG_MUTEX.synchronize do
+      Log::LAST.replace "log"
+    end
+    nil
   end
 
   def self.stack(stack)

data/lib/rbbt/util/misc/bgzf.rb

@@ -29,8 +29,8 @@ module Bgzf
 
   def filename
     @filename ||= begin
-                  compressed_stream.respond_to?(:filename) ? compressed_stream.filename : nil
-                end
+                    compressed_stream.respond_to?(:filename) ? compressed_stream.filename : rand(1000000000).to_s
+                  end
   end
 
   def closed?
@@ -49,7 +49,8 @@ module Bgzf
 
   def _index
     @_index ||= begin
-                  index = Persist.persist("BGZF index" + (filename || "").sub(/.bgz$/,''), :marshal, :dir => Rbbt.var.bgzf_index) do
+                  prefix_code = "BGZF index" + (filename || "").sub(/.bgz$/,'')
+                  index = Persist.persist(prefix_code, :marshal) do
                     index = []
                     pos = 0
                     while true do
@@ -66,7 +67,7 @@ module Bgzf
                     end
                     index
                   end
-                  @block_cache_size = Math.log(index.length).to_i + 1
+                  @block_cache_size = Math.log(index.length + 1).to_i + 1
                   index
                end
   end

data/lib/rbbt/util/misc/development.rb

@@ -39,13 +39,13 @@ module Misc
         end
       end
       if message
-        puts "#{message }: #{ repeats } repeats"
+        STDERR.pust "#{message }: #{ repeats } repeats"
       else
-        puts "Benchmark for #{ repeats } repeats"
+        STDERR.puts "Benchmark for #{ repeats } repeats"
       end
-      puts measure
+      STDERR.puts measure
     rescue Exception
-      puts "Benchmark aborted"
+      STDERR.puts "Benchmark aborted"
       raise $!
     end
     res

data/lib/rbbt/util/misc.rb

@@ -69,7 +69,7 @@ module Misc
   end
 
   def self.tokenize(str)
-    str.scan(/"[^"]*"|'[^']*'|[^"'\s]+/)
+    str.scan(/"([^"]*)"|'([^']*)'|([^"'\s]+)/).flatten.compact
   end
 
   def self.timespan(str, default = "s")

data/lib/rbbt/util/named_array.rb

@@ -203,7 +203,7 @@ module NamedArray
 
   def report
     fields.zip(self).collect do |field,value|
-      "#{ field }: #{ Array === value ? value * "|" : value }"
+      "#{Log.color(:magenta, field) }: #{ Array === value ? value * "|" : value }"
     end * "\n"
   end
 

data/lib/rbbt/util/open.rb

@@ -101,7 +101,6 @@ module Open
 
       CMD.cmd("wget '#{ url }'", wget_options)
     rescue
-     STDERR.puts $!.backtrace.inspect
      raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
     end
   end
@@ -654,7 +653,12 @@ module Open
   end
 
   def self.download(url, path)
-    Open.wget(url, "--output-document" => path, :pipe => false)
+    begin
+      Open.wget(url, "--output-document" => path, :pipe => false)
+    rescue Exception
+      Open.rm(path) if Open.exist?(path)
+      raise $!
+    end
   end
 
   def self.can_open?(file)

data/lib/rbbt/workflow/definition.rb

@@ -129,6 +129,8 @@ module Workflow
     end
   end
 
+  alias task_alias dep_task
+
   def task(name, &block)
     if Hash === name
       type = name.first.last