rbbt-util 5.42.0 → 5.44.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b23de77e3900d1a81e8ffa349ab9f88a4283f30bb2cec62c04bff4539243170
-  data.tar.gz: 494b18d5e2ba4802225617a03b2f69255ec9deab173e8d18e06654a7b415325c
+  metadata.gz: 1c39baa8ee0c2633bea3878720d29aa0f49a057db7ec6700187685c7a07f7eae
+  data.tar.gz: ad0f8a09cb34faa3d0aa3388c26733d4849d307a02df02989656c0aced5ea72a
 SHA512:
-  metadata.gz: d57bd3bc7ff84a4e8101dd907f98ef20923dbcab95c3187523e6b84eba6774d900f269ed2457fd6e1a81821edf3e2d1f11fcbdd5863f791282d549bcf83a48a5
-  data.tar.gz: a4857d82292ab4234cf10fbf2fcb242b466e6400f9476cb4de4bc625ff808542a690f27fbd9dede4ff692a82644561ebfc086ccdf953102cb4f7a0020b094849
+  metadata.gz: dcfcf7f6ea2b45dd4ca5e1257fb22cec0f06280be2b6c9aac3b118e1e649a8f5948db0162f62735b9c113ee9a878cb89e8f98a99fd7026a7f58f061531b93f89
+  data.tar.gz: 59e94fecbb50bf9d7635545aab508ee6b986e44f81acac59aac7e888f6c9c5748fb2d0324a0e83d784c78b249e67a50baaaba1d40b58612f194de111fd4c6579

data/lib/rbbt/annotations/util.rb

@@ -43,7 +43,7 @@ module Annotated
   end
 
   def self.resolve_array(entry)
-    if entry =~ /^Array:/
+    if String === entry && entry =~ /^Array:/
       entry["Array:".length..-1].split("|")
     else
       entry

data/lib/rbbt/association/database.rb

@@ -93,7 +93,7 @@ module Association
       tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
 
       tsv.with_monitor(options[:monitor]) do
-        tsv = tsv.reorder source_field, fields, :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
+        tsv = tsv.reorder source_field, tsv.all_fields.values_at(*field_pos), :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
       end
 
       tsv.key_field = source_header
@@ -216,6 +216,7 @@ module Association
                  open_stream(stream, options.dup)
                end
 
+    database.filename = file if Path === file && file.identifier_files.any?
     database.entity_options = options[:entity_options] if options[:entity_options]
 
     database

data/lib/rbbt/association/index.rb

@@ -38,6 +38,7 @@ module Association
       data.fields = fields[1..-1]
       data.type = :list
       data.serializer ||= serializer
+      data.filename ||= file if String === file
 
       database.with_unnamed do
         database.with_monitor(options[:monitor]) do

data/lib/rbbt/association/util.rb

@@ -10,7 +10,7 @@ module Association
   end
 
   def self.parse_field_specification(spec)
-    return [2,nil,nil] if Numeric === spec
+    return [spec,nil,nil] if Numeric === spec
     spec = spec.split "=>" unless Array === spec
     field_part, final_format = spec
 
@@ -115,8 +115,14 @@ module Association
     source_field = specs[:source][0]
     target_field = specs[:target][0]
 
-    source_pos = all_fields.index source_field
-    target_pos = all_fields.index target_field
+    #source_pos = all_fields.index source_field
+    #target_pos = all_fields.index target_field
+   
+    source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
+    target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
+
+    source_pos = source_pos == :key ? 0 : source_pos + 1
+    target_pos = target_pos == :key ? 0 : target_pos + 1
 
     source_header = specs[:source][1] || specs[:source][0]
     target_header = specs[:target][1] || specs[:target][0]
@@ -140,7 +146,11 @@ module Association
       field_headers << header
     end
 
-    field_pos = info_fields.collect{|f| raise "Field #{f} not found. Options: #{all_fields * ", "}" unless all_fields.include?(f); f == :key ? 0 : all_fields.index(f);  }
+    field_pos = info_fields.collect do |f| 
+      p = TSV.identify_field all_fields.first, all_fields[1..-1], f
+      p == :key ? 0 : p + 1
+    end
+
     field_pos.delete source_pos
 
     source_format = specs[:source][2]

data/lib/rbbt/entity.rb

@@ -289,16 +289,17 @@ module Entity
       end
 
       def self.unpersist(method_name)
-        return unless persisted? method_name
-        orig_name = UNPERSISTED_PREFIX + method_name.to_s
-
+        return unless orig_name = persisted?(method_name)
         alias_method method_name, orig_name
         remove_method orig_name
       end
 
       def self.persisted?(method_name)
-        orig_name = UNPERSISTED_PREFIX + method_name.to_s
-        instance_methods.include? orig_name.to_sym
+        ["", "_multiple_", "_single_"].each do |type|
+          orig_name = (UNPERSISTED_PREFIX + type + method_name.to_s).to_sym
+          return orig_name if instance_methods.include?(orig_name)
+        end
+        return false
       end
 
       def self.with_persisted(method_name)

data/lib/rbbt/hpc/batch.rb

@@ -110,23 +110,25 @@ module HPC
 
       task = Symbol === job.overriden ? job.overriden : job.task_name
 
-      #override_deps = job.overriden_deps.collect do |dep| 
-      #  name = [dep.workflow.to_s, dep.task_name] * "#"
-      #  [name, dep.path] * "="  
-      #end.uniq * ","
-
-      if job.overriden?
-        #override_deps = job.rec_dependencies.
-        #  select{|dep| Symbol === dep.overriden }.
-        
-        override_deps = job.overriden_deps.
-          collect do |dep| 
-          name = [dep.workflow.to_s, dep.task_name] * "#"
-          [name, dep.path] * "="  
-        end.uniq * ","
-
-        options[:override_deps] = override_deps unless override_deps.empty?
-      end
+      override_deps = job.overriden_deps.collect do |dep| 
+        name = [dep.workflow.to_s, dep.task_name] * "#"
+        [name, dep.path] * "="  
+      end.uniq * ","
+
+      options[:override_deps] = override_deps unless override_deps.empty?
+
+      #if job.overriden?
+      #  #override_deps = job.rec_dependencies.
+      #  #  select{|dep| Symbol === dep.overriden }.
+      #  
+      #  override_deps = job.overriden_deps.
+      #    collect do |dep| 
+      #    name = [dep.workflow.to_s, dep.task_name] * "#"
+      #    [name, dep.path] * "="  
+      #  end.uniq * ","
+
+      #  options[:override_deps] = override_deps unless override_deps.empty?
+      #end
 
       # Save inputs into inputs_dir
       inputs_dir = Misc.process_options options, :inputs_dir

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -152,7 +152,7 @@ module HPC
     end
 
     def self.job_batches(rules, job)
-      job_chains = self.job_chains(rules, job)
+      job_chains = self.job_chains(rules, job).dup
 
       workload = job_workload(job).uniq
 

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -39,9 +39,8 @@ module HPC
       (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
     end
 
-    def self.job_chains(rules, job)
-      @@job_chains ||= {}
-      @@job_chains[Misc.digest([rules, job.path].inspect)] ||= 
+    def self.job_chains(rules, job, computed = {})
+      computed[Misc.fingerprint([rules, job.path, job.object_id])] ||=
         begin
           chains = self.parse_chains(rules)
 
@@ -55,7 +54,7 @@ module HPC
             dep_matches = check_chains(chains, dep)
             common = matches & dep_matches
 
-            dep_chains = job_chains(rules, dep)
+            dep_chains = job_chains(rules, dep, computed)
             found = []
             dep_chains.each do |match,info|
               if common.include?(match)

data/lib/rbbt/knowledge_base/traverse.rb

@@ -146,7 +146,7 @@ class KnowledgeBase
       if conditions
         Misc.tokenize(conditions).each do |condition|
           if condition.index "="
-            key, value = conditions.split("=")
+            key, value = condition.split("=")
             matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
           else
             matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
@@ -158,17 +158,19 @@ class KnowledgeBase
     end
 
     def id_dbs(db)
+      # ToDo: Revise this, I'm not sure what id does anymore
+      # I think it deals with syndication
       if db.include? '?'
-        all_dbs = kb.registry.keys
+        all_dbs = kb.registry.keys.collect{|k| k.to_s }
         _name, _sep, _kb = db.partition("@")
         case
         when _name[0] == '?'
           dbs = all_dbs.select{|_db| 
             n,_s,d=_db.partition("@"); 
-            d.nil? or d.empty? or (d == _kd and assignments[_name].include?(n))
+            d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
           }
         when _kb[0] == '?'
-          dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d)}
+          dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
         end
       else
         dbs = [db]
@@ -181,11 +183,12 @@ class KnowledgeBase
       all_matches = []
       path_rules = []
       acc_var = nil
+      pre_acc_var_assignments = nil
       rules.each do |rule|
         rule = rule.strip
         next if rule.empty?
 
-        if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/)
+        if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
           Log.debug "Traverse rule: #{rule}"
           path_rules << rule
 
@@ -199,17 +202,19 @@ class KnowledgeBase
 
             next if matches.nil? or matches.empty?
 
-            if db.include? '?'
-              _name, _sep, _kb = db.partition("@")
-              case
-              when _kb[0] == '?'
-                assignments[_kb] ||= []
-                assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
-              when _name[0] == '?'
-                assignments[_name] ||= []
-                assignments[_name] << _db.partition("@").first
-              end
-            end
+            # ToDo: Revise this, I'm not sure what id does anymore
+            #
+            #if db.include? '?'
+            #  _name, _sep, _kb = db.partition("@")
+            #  case
+            #  when _kb[0] == '?'
+            #    assignments[_kb] ||= []
+            #    assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
+            #  when _name[0] == '?'
+            #    assignments[_name] ||= []
+            #    assignments[_name] << _db.partition("@").first
+            #  end
+            #end
 
             matches.each do |m|
               rule_matches << m
@@ -244,11 +249,14 @@ class KnowledgeBase
 
         elsif m = rule.match(/(\?[^\s{]+)\s*{/)
           acc_var = m.captures.first
+          pre_acc_var_assignments = assignments.dup
           Log.debug "Start assign block: #{acc_var}"
         elsif m = rule.match(/^\s*}\s*$/)
           Log.debug "Close assign block: #{acc_var}"
           saved_assign = assignments[acc_var]
           assignments.clear
+          assignments.merge!(pre_acc_var_assignments)
+          pre_acc_var_assignments = nil
           assignments[acc_var] = saved_assign
           all_matches = []
           path_rules = []

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -21,7 +21,7 @@ module Persist
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-      database = CONNECTIONS[path] ||= tokyocabinet_class.new
+      database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
 
       if big and not Open.exists?(path)
         database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 

data/lib/rbbt/resource/path.rb

@@ -161,8 +161,13 @@ module Path
   def find(where = nil, caller_lib = nil, paths = nil)
 
     if located?
-      self.original ||= self
-      return self
+      path = self
+      path = path + '.gz' if File.exist?(path + '.gz')
+      path = path + '.bgz' if File.exist?(path + '.bgz')
+      self.annotate(path)
+
+      path.original = self
+      return path
     end
 
     if where == :all || where == 'all'

data/lib/rbbt/resource.rb

@@ -374,7 +374,7 @@ url='#{url}'
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
             unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
             unlocated.gsub!(/\/+/,'/')
-            if self.subdir && ! self.subdir.empty?
+            if self.subdir && ! self.subdir.empty? && unlocated.include?(subdir)
               subdir = self.subdir
               subdir += "/" unless subdir.end_with?("/")
               unlocated[subdir] = ""

data/lib/rbbt/tsv/accessor.rb

@@ -567,13 +567,13 @@ module TSV
     end
   end
 
-  def dumper_stream(keys = nil, no_options = false, unmerge = false)
+  def dumper_stream(keys = nil, no_options = false, unmerge = false, stream = nil)
     unmerge = false unless type == :double
 
     options = self.options
     options[:type] = :list if unmerge
 
-    TSV::Dumper.stream options do |dumper|
+    TSV::Dumper.stream options, filename, stream do |dumper|
       case no_options
       when FalseClass, nil
         dumper.init
@@ -631,11 +631,11 @@ module TSV
             end
           end
         end
+        dumper.close
       rescue Exception
         Log.exception $!
         raise $!
       end
-      dumper.close
     end
   end
 
@@ -651,14 +651,9 @@ module TSV
       end
     end
 
-    io = dumper_stream(keys, no_options, unmerge)
-
-    str = ''
-    while block = io.read(Misc::BLOCK_SIZE)
-      str << block
-    end
-
-    str
+    io = dumper_stream(keys, no_options, unmerge, StringIO.new)
+    io.rewind
+    io.read
   end
 
   def to_unmerged_s(keys = nil, no_options = false)

data/lib/rbbt/tsv/dumper.rb

@@ -1,13 +1,24 @@
 module TSV
   class Dumper
     attr_accessor :in_stream, :stream, :options, :filename, :sep
-    def self.stream(options = {}, filename = nil, &block)
+    def self.stream(options = {}, filename = nil, stream = nil, &block)
       dumper = TSV::Dumper.new options, filename
-      Thread.new(Thread.current) do |parent|
+      if stream
+        dumper.set_stream stream if stream
         yield dumper
-        dumper.close
+        stream
+      else
+        thread = Thread.new(Thread.current) do |parent|
+          yield dumper
+        end
+        ConcurrentStream.setup(dumper.stream, threads: thread)
       end
-      dumper.stream
+    end
+
+    def set_stream(stream)
+      @stream.close
+      @in_stream.close
+      @in_stream = @stream = stream
     end
 
     def initialize(options, filename = nil)
@@ -56,16 +67,16 @@ module TSV
 
       str = TSV.header_lines(key_field, fields, options.merge(init_options || {}))
 
-      Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
+      Thread.pass while IO.select(nil, [@in_stream],nil,1).nil? if IO === @in_stream
 
-      @in_stream.puts str
+      @in_stream << str
     end
 
     def add(k,v)
       @fields ||= @options[:fields]
       @sep ||= @options[:sep]
       begin
-        Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
+        Thread.pass while IO.select(nil, [@in_stream],nil,1).nil? if IO === @in_stream
         @in_stream << k << TSV::Dumper.values_to_s(v, @fields, @sep)
       rescue IOError
       rescue Exception
@@ -74,16 +85,16 @@ module TSV
     end
 
     def close_out
-      @stream.close unless @stream.closed?
+      @stream.close unless StringIO === @stream || @stream.closed?
     end
 
     def close_in
       @in_stream.join if @in_stream.respond_to?(:join) && ! @in_stream.joined?
-      @in_stream.close unless @in_stream.closed?
+      @in_stream.close if @in_stream.respond_to?(:close) && ! @in_stream.closed? 
     end
 
     def close
-      close_in
+      close_in unless @in_stream == @stream
     end
   end
 end

data/lib/rbbt/tsv/index.rb

@@ -125,7 +125,7 @@ module TSV
   def self.index(file, options = {})
     persist_options = Misc.pull_keys options, :persist
     persist_options[:prefix] ||= "StaticIndex[#{options[:target] || :key}]"
-     
+
     Log.debug "Static Index: #{ file } - #{Misc.fingerprint options}"
     Persist.persist_tsv nil, file, options, persist_options do |data|
       data_options = Misc.pull_keys options, :data
@@ -258,6 +258,7 @@ module TSV
                else
                  file.object_id.to_s
                end
+
     persist_options = Misc.pull_keys options, :persist
     persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -181,6 +181,48 @@ module TSV
     end
   end
 
+  def self.traverse_enumerable(enum, options = {}, &block)
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
+
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        while enum.any?
+          e = enum.pop
+          begin
+            callback.call yield(e)
+          rescue Exception
+            Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
+            raise $!
+          ensure
+            bar.tick if bar
+          end
+        end
+      else
+        bar.init if bar
+        while enum.any?
+          e = enum.pop
+          begin
+            yield e
+          rescue Exception
+            Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
+            raise $!
+          ensure
+            bar.tick if bar
+          end
+        end
+      end
+
+    rescue
+      error = true
+      raise $!
+    ensure
+      join.call(error) if join
+      Log::ProgressBar.remove_bar(bar, error) if bar
+    end
+  end
+
   def self.traverse_priority_queue(queue, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
@@ -396,6 +438,8 @@ module TSV
         else
           raise "Can not open obj for traversal #{Misc.fingerprint obj}"
         end
+      when Enumerable
+        traverse_enumerable(obj, options, &block)
       when nil
         raise "Can not traverse nil object into #{stream_name(options[:into])}"
       else

data/lib/rbbt/tsv/parser.rb

@@ -281,19 +281,20 @@ module TSV
       end
 
       values = values.collect{|v| v.length != num ? [v.first] * num : v}
+
       all = values
       all.unshift keys
-      Misc.zip_fields(all).each do |values|
-        key = values.shift
+      Misc.zip_fields(all).each do |vs|
+        key = vs.shift
         if data.include? key
-          data[key] = data[key].zip(values).collect do |old, new|
-            old.push new
-            old
+          data[key] = data[key].zip(vs).collect do |old, new|
+            old + [new]
           end
         else
-          data[key] = values.collect{|v| [v]}
+          data[key] = vs.collect{|v| [v] }
         end
       end
+
       nil
     end
 

data/lib/rbbt/tsv/serializers.rb

@@ -63,7 +63,7 @@ module TSV
   class StringDoubleArraySerializer
     def self.dump(array)
       begin
-      array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
+        array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
       rescue Encoding::CompatibilityError
         array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
       end

data/lib/rbbt/tsv/util.rb

@@ -168,6 +168,8 @@ module TSV
       end
     when Set
       get_stream(file.to_a, open_options)
+    when Enumerable
+      file
     else
       raise "Cannot get stream from: #{file.inspect}"
     end
@@ -189,6 +191,16 @@ module TSV
       pos = fields.index field
       return pos if pos
       return identify_field(key_field, fields, field.to_i) if field =~ /^\d+$/
+      if fields.select{|f| f.include?("(") }.any?
+        simplify_fields = fields.collect do |f|
+          if m = f.match(/(.*)\s+\(.*\)/)
+            m[1]
+          else
+            f
+          end
+        end
+        return identify_field(key_field, simplify_fields, field)
+      end
       raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}" if pos.nil?
     else
       raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}"

data/lib/rbbt/util/log/progress.rb

@@ -9,7 +9,7 @@ module Log
 
   def self.no_bar
     @@no_bar = false unless defined?(@@no_bar)
-    @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
+    (@@no_bar || ENV["RBBT_NO_PROGRESS"]).to_s == "true"
   end
 
   class ProgressBar

data/lib/rbbt/util/misc/bgzf.rb

@@ -29,8 +29,8 @@ module Bgzf
 
   def filename
     @filename ||= begin
-                  compressed_stream.respond_to?(:filename) ? compressed_stream.filename : nil
-                end
+                    compressed_stream.respond_to?(:filename) ? compressed_stream.filename : rand(1000000000).to_s
+                  end
   end
 
   def closed?
@@ -49,7 +49,8 @@ module Bgzf
 
   def _index
     @_index ||= begin
-                  index = Persist.persist("BGZF index" + (filename || "").sub(/.bgz$/,''), :marshal, :dir => Rbbt.var.bgzf_index) do
+                  prefix_code = "BGZF index" + (filename || "").sub(/.bgz$/,'')
+                  index = Persist.persist(prefix_code, :marshal) do
                     index = []
                     pos = 0
                     while true do
@@ -66,7 +67,7 @@ module Bgzf
                     end
                     index
                   end
-                  @block_cache_size = Math.log(index.length).to_i + 1
+                  @block_cache_size = Math.log(index.length + 1).to_i + 1
                   index
                end
   end

data/lib/rbbt/util/misc/development.rb

@@ -39,13 +39,13 @@ module Misc
         end
       end
       if message
-        puts "#{message }: #{ repeats } repeats"
+        STDERR.pust "#{message }: #{ repeats } repeats"
       else
-        puts "Benchmark for #{ repeats } repeats"
+        STDERR.puts "Benchmark for #{ repeats } repeats"
       end
-      puts measure
+      STDERR.puts measure
     rescue Exception
-      puts "Benchmark aborted"
+      STDERR.puts "Benchmark aborted"
       raise $!
     end
     res

data/lib/rbbt/util/misc/math.rb

@@ -1,4 +1,3 @@
-
 module Misc
 
   Log2Multiplier = 1.0 / Math.log(2.0)

data/lib/rbbt/util/misc.rb

@@ -69,7 +69,7 @@ module Misc
   end
 
   def self.tokenize(str)
-    str.scan(/"[^"]*"|'[^']*'|[^"'\s]+/)
+    str.scan(/"([^"]*)"|'([^']*)'|([^"'\s]+)/).flatten.compact
   end
 
   def self.timespan(str, default = "s")

data/lib/rbbt/util/open.rb

@@ -729,7 +729,7 @@ module Open
       dir_sub_path.push content
       save_content_in_repo(*dir_sub_path)
     else
-      FileUtils.mkdir_p File.dirname(file)
+      FileUtils.mkdir_p File.dirname(file) unless File.directory?(file)
       case
       when block_given?
         begin