rbbt-util 5.34.7 → 5.34.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec5860171e51e2e53a8383ae8ea1156b79c7f75194d384c5aa2c43efd759e129
-  data.tar.gz: a6ab8a782b3169eae1e475a2029e8a4f0ac2d12f375475ec70de9d97f513f21a
+  metadata.gz: c8d689458bf34dc4d6f87f0541579a39ce1820470028db527ea366ad9ebb5c60
+  data.tar.gz: 4adc7dfec047abdeff9783386c2d0c017b37e4459b51d928179dc9199a779b5c
 SHA512:
-  metadata.gz: 2fb6a8ac5abee267891f6bb0736b659f202019de75fa0ecef80e955838a54013eaeabb495c44daff45196a8706821b706bfae75b095ce3ca8b82e8351436d4ef
-  data.tar.gz: ab24e4ab588e75f94adfcd9a02e178511bd4471edadef18c6c49537974353b8b4d83c61438d8fe638acbc26101556637a92d5da325ebc135736da44558971334
+  metadata.gz: ec17b39479ed23fabae816d724281a9228ca0fcced777cb3d22c23a3a04e136e132b67cb7e0b5c82aec1e754253983f8912f5d7af4ab52d80cb3d3e0464f8fd2
+  data.tar.gz: a61457b87e1ce1243c903e46bf06f88242fbb6e939eb44f9fb23a8531c24e023471cbb2d42ecd7f3886eb7efa52d333ac94df348a0f8e29be0eecacc4d5d0ae8

data/lib/rbbt/hpc/batch.rb

@@ -514,7 +514,7 @@ env > #{batch_options[:fenv]}
 
       %w(std.out std.err job.id job.status dependencies.list canfail_dependencies.list exit.status sync.log inputs_dir).each do |filename|
         path = File.join(batch_dir, filename)
-        Open.rm_rf path if File.exists? path
+        Open.rm_rf path if File.exist? path
       end if clean_batch_job
 
       batch_dependencies = [] if batch_dependencies.nil?
@@ -584,14 +584,14 @@ env > #{batch_options[:fenv]}
 
       job = Open.read(fjob).strip if Open.exists?(fjob)
 
-      if job && ! File.exists?(fexit)
+      if job && ! File.exist?(fexit)
         begin
           status_txt = job_status(job)
           STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
           STDERR.puts status_txt
           lines = status_txt.split("\n").length
         rescue
-          if ! File.exists?(fexit)
+          if ! File.exist?(fexit)
             STDERR.puts Log.color(:magenta, "Job #{job.to_i} not done and not running. STDERR:")
             STDERR.puts Open.read(ferr)
           end
@@ -599,7 +599,7 @@ env > #{batch_options[:fenv]}
         end
       end
 
-      if File.exists?(fexit)
+      if File.exist?(fexit)
         exit_status = Open.read(fexit)
         if exit_status.to_i == 0
           STDERR.puts Log.color(:magenta, "Job #{job} done with exit_status 0. STDOUT:")
@@ -613,7 +613,7 @@ env > #{batch_options[:fenv]}
 
       if tail
         Log.severity = 10
-        while ! File.exists? fout
+        while ! File.exist? fout
           if job
             STDERR.puts
             Log.clear_line(STDERR)
@@ -638,8 +638,8 @@ env > #{batch_options[:fenv]}
         begin
           status_txt = job_status(job)
           Open.write(fstatus, status_txt) unless status_txt.nil? || status_txt.empty?
-          out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
-          err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
+          out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exist?(fout) and not tail == :STDERR
+          err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exist?(ferr)
 
           terr = Misc.consume_stream(err, true, STDERR) if err
           tout = Misc.consume_stream(out, true, STDOUT) if out

data/lib/rbbt/hpc/lsf.rb

@@ -70,12 +70,12 @@ export BATCH_SYSTEM=LSF
       STDERR.puts Log.color(:magenta, "Issuing LSF file: #{fcmd}")
       STDERR.puts Open.read(fcmd)
 
-      if File.exists?(fjob)
+      if File.exist?(fjob)
         job = Open.read(fjob).to_i
       else
 
-        dependencies = Open.read(fdep).split("\n") if File.exists? fdep
-        canfail_dependencies = Open.read(fcfdep).split("\n") if File.exists? fcfdep
+        dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+        canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
 
         normal_dep_list = dependencies && dependencies.any? ? dependencies.collect{|d| "post_done(#{d})"} : []
         canfail_dep_list = canfail_dependencies && canfail_dependencies.any? ? canfail_dependencies.collect{|d| "done(#{d})"} : []
@@ -90,7 +90,7 @@ export BATCH_SYSTEM=LSF
 
         cmd = "bsub #{dep_str} < '#{fcmd}'"
 
-        if File.exists?(fout)
+        if File.exist?(fout)
           return
         elsif dry_run
           STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, cmd)

data/lib/rbbt/hpc/slurm.rb

@@ -101,12 +101,12 @@ export BATCH_SYSTEM=SLURM
       Log.info "Issuing SLURM file: #{fcmd}"
       Log.debug Open.read(fcmd)
 
-      if File.exists?(fjob)
+      if File.exist?(fjob)
         job = Open.read(fjob).to_i
       else
 
-        dependencies = Open.read(fdep).split("\n") if File.exists? fdep
-        canfail_dependencies = Open.read(fcfdep).split("\n") if File.exists? fcfdep
+        dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+        canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
 
         normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
         canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
@@ -119,7 +119,7 @@ export BATCH_SYSTEM=SLURM
 
         cmd = "sbatch #{dep_str} '#{fcmd}'"
 
-        if File.exists?(fout)
+        if File.exist?(fout)
           return
         elsif dry_run
           STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "sbatch '#{fcmd}'")

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -16,7 +16,7 @@ module Persist
       end
 
       dir = File.dirname(File.expand_path(path))
-      File.mkdir(dir) unless File.exists?(dir)
+      File.mkdir(dir) unless File.exist?(dir)
 
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"

data/lib/rbbt/persist.rb

@@ -31,8 +31,8 @@ module Persist
     path = path.find if Path === path
     file = file.find if Path === file
     if by_link
-      patht = File.exists?(path) ? File.lstat(path).mtime : nil
-      filet = File.exists?(file) ? File.lstat(file).mtime : nil
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
     else
       patht = Open.mtime(path)
       filet = Open.mtime(file)

data/lib/rbbt/resource.rb

@@ -233,7 +233,7 @@ module Resource
             when :install
               Log.debug "Installing software: #{path}"
 
-              $set_software_env = false unless File.exists? path
+              $set_software_env = false unless File.exist? path
               
               software_dir = path.resource.root.software.find :user
               helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))

data/lib/rbbt/tsv/manipulate.rb

@@ -527,13 +527,21 @@ module TSV
         self.unnamed = old_unnamed
 
       when String === method
-        with_unnamed do
-          through :key, key do |key, values|
-            values = [values] if type == :single
-            new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.length > 0)
+        if method =~ /^([<>]=?)(.*)/
+          with_unnamed do
+            through :key, key do |key, values|
+              value = Array === values ? values.flatten.first : values
+              new[key] = self[key] if value.to_f.send($1, $2.to_f)
+            end
+          end
+        else
+          with_unnamed do
+            through :key, key do |key, values|
+              values = [values] if type == :single
+              new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.length > 0)
+            end
           end
         end
-
       when Numeric === method
         with_unnamed do
           through :key, key do |key, values|

data/lib/rbbt/util/config.rb

@@ -145,7 +145,7 @@ module Rbbt::Config
   end
 
   def self.process_config(config)
-    if Misc.is_filename?(config) && File.exists?(config)
+    if Misc.is_filename?(config) && File.exist?(config)
       Rbbt::Config.load_file(config)
     elsif Rbbt.etc.config_profile[config].exists?
       Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)

data/lib/rbbt/util/log/progress.rb

@@ -49,6 +49,9 @@ module Log
     end
 
     def init
+      @ticks, @bytes = 0
+      @last_time = @last_count = @last_percent = nil
+      @history, @mean_max, @max_history = nil
       @start = @last_time = Time.now
       @last_count = 0
       report

data/lib/rbbt/util/migrate.rb

@@ -17,7 +17,7 @@ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
 
       [path, paths.collect{|p| [source, p] * ":"}, lpath]
     else
-      if File.exists?(path)
+      if File.exist?(path)
         path = resource.identify(path)
       else
         path = Path.setup(path)

data/lib/rbbt/util/misc/inspect.rb

@@ -266,7 +266,7 @@ module Misc
 
   def self.mtime_str(path)
     path = path.find if Path === path
-    if File.exists? path
+    if File.exist? path
       "mtime: " << File.mtime(path).to_s
     else
       "mtime: not present"
@@ -416,7 +416,7 @@ module Misc
   end
 
   def self.file2md5(file)
-    if File.exists?(file + '.md5')
+    if File.exist?(file + '.md5')
       Open.read(file + '.md5')
     else
       md5 = CMD.cmd("md5sum '#{file}'").read.strip.split(" ").first

data/lib/rbbt/util/misc/pipes.rb

@@ -39,11 +39,11 @@ module Misc
     begin
       erase = path.nil?
       path = TmpFile.tmp_file if path.nil?
-      File.rm path if clean && File.exists?(path)
+      File.rm path if clean && File.exist?(path)
       File.mkfifo path
       yield path
     ensure
-      FileUtils.rm path if erase && File.exists?(path)
+      FileUtils.rm path if erase && File.exist?(path)
     end
   end
   
@@ -340,11 +340,11 @@ module Misc
         Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort
         #io.close unless io.closed?
-        FileUtils.rm into_path if into_path and File.exists? into_path
+        FileUtils.rm into_path if into_path and File.exist? into_path
       rescue Exception
         Log.medium "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
         io.abort $! if io.respond_to? :abort
-        FileUtils.rm into_path if into_path and File.exists? into_path
+        FileUtils.rm into_path if into_path and File.exist? into_path
         raise $!
       end
     end

data/lib/rbbt/util/open.rb

@@ -389,8 +389,8 @@ module Open
   #  source = source.find if Path === source
   #  target = target.find if Path === target
 
-  #  FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-  #  FileUtils.rm target if File.exists?(target)
+  #  FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+  #  FileUtils.rm target if File.exist?(target)
   #  FileUtils.cp source, target
   #end
 

data/lib/rbbt/util/tar.rb

@@ -93,7 +93,7 @@ module Misc
   # directory
   def self.untar(io, destination)
     io = io.find if Path === io
-    if String === io and File.exists?(io)
+    if String === io and File.exist?(io)
       Open.open(io) do |f|
         untar(f, destination)
       end

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -13,7 +13,7 @@ module Workflow
     nextflow_includes(file).inject(params) do |params,info|
       name_str, included_file = info
       included_file = File.join(dir, included_file)
-      included_file += '.nf' unless File.exists?(included_file) || ! File.exists?(included_file + '.nf')
+      included_file += '.nf' unless File.exist?(included_file) || ! File.exist?(included_file + '.nf')
       name_str.split(";").each do |name|
         name = name.strip
         include_params = nextflow_recursive_params(included_file).collect{|p| [p,name] * "-"}
@@ -32,7 +32,7 @@ module Workflow
       result = :text
     end
 
-    file = file + '.nf' unless File.exists?(file) || ! File.exists?(file + '.nf')
+    file = file + '.nf' unless File.exist?(file) || ! File.exist?(file + '.nf')
     file = File.expand_path(file)
     name ||= File.basename(file).sub(/\.nf$/,'').gsub(/\s/,'_')
     params = Workflow.nextflow_recursive_params(file)

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,7 @@ class Step
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
         if ! (job.resumable? && (job.updated? && ! job.dirty?))
-          Log.high "About to clean -- status: #{status}, present #{File.exists?(job.path)}, " +
+          Log.high "About to clean -- status: #{status}, present #{File.exist?(job.path)}, " +
             %w(done? error? recoverable_error? noinfo? updated? dirty? aborted? running? resumable?).
             collect{|v| [v, job.send(v)]*": "} * ", " if RBBT_DEBUG_CLEAN
 

data/lib/rbbt/workflow/step/run.rb

@@ -421,7 +421,7 @@ class Step
           config_keys = Rbbt::Config::GOT_KEYS[config_keys_pre.length..-1]
           set_info :config_keys, config_keys.uniq
 
-          if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
+          if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
             Open.mv self.tmp_path, self.path
           end
           result

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -2,12 +2,12 @@ module Workflow
 
   def self.load_inputs(dir, input_names, input_types)
     inputs = {}
-    if File.exists?(dir) && ! File.directory?(dir)
+    if File.exist?(dir) && ! File.directory?(dir)
       Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
       tarfile = dir
       digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
       tmpdir = Rbbt.tmp.input_bundle[digest].find
-      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+      Misc.untar(tarfile, tmpdir) unless File.exist? tmpdir
       files = tmpdir.glob("*")
       if files.length == 1 && File.directory?(files.first)
         tmpdir = files.first
@@ -256,7 +256,7 @@ class Step
   #    when Step === value
   #      Open.ln_s(value.path, path)
   #    when type.to_s == "binary"
-  #      if String === value && File.exists?(value)
+  #      if String === value && File.exist?(value)
   #        value = File.expand_path(value)
   #        Open.ln_s(value, path)
   #      elsif String === value && Misc.is_filename?(value, false)
@@ -269,7 +269,7 @@ class Step
   #    when Array === value
   #      Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
   #    when %w(file tsv array).include?(type.to_s)
-  #      if String === value && File.exists?(value)
+  #      if String === value && File.exist?(value)
   #        value = File.expand_path(value)
   #        Open.ln_s(value, path)
   #      elsif String === value && Misc.is_filename?(value, false)
@@ -280,7 +280,7 @@ class Step
   #        Open.write(path + '.yaml', value.to_yaml)
   #      end
   #    when IO === value
-  #      if value.filename && String === value.filename && File.exists?(value.filename)
+  #      if value.filename && String === value.filename && File.exist?(value.filename)
   #        Open.ln_s(value.filename, path)
   #      else
   #        Open.write(path, value)

data/lib/rbbt/workflow/util/archive.rb

@@ -13,9 +13,9 @@ class Step
     return if target_dir[workflow][task][name].exists? || File.symlink?(target_dir[workflow][task][name].find)
     Log.debug "Linking #{ path }"
     FileUtils.mkdir_p target_dir[workflow][task] unless target_dir[workflow][task].exists?
-    FileUtils.ln_s path, target_dir[workflow][task][name].find if File.exists?(path)
-    FileUtils.ln_s path + '.files', target_dir[workflow][task][name].find + '.files' if File.exists?(path + '.files')
-    FileUtils.ln_s path + '.info', target_dir[workflow][task][name].find + '.info' if File.exists?(path + '.info')
+    FileUtils.ln_s path, target_dir[workflow][task][name].find if File.exist?(path)
+    FileUtils.ln_s path + '.files', target_dir[workflow][task][name].find + '.files' if File.exist?(path + '.files')
+    FileUtils.ln_s path + '.info', target_dir[workflow][task][name].find + '.info' if File.exist?(path + '.info')
   end
 
   def archive(target = nil)
@@ -65,15 +65,15 @@ class Step
     end.uniq
 
     jobs.each do |step|
-      next unless File.exists?(step.path)
+      next unless File.exist?(step.path)
       next if skip_overriden && step.overriden
 
       job_files << step.path
-      job_files << step.info_file if File.exists?(step.info_file)
-      job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
+      job_files << step.info_file if File.exist?(step.info_file)
+      job_files << Step.md5_file(step.path) if File.exist?(Step.md5_file step.path)
       job_file_dir_content = Dir.glob(step.files_dir + '/**/*')
       job_files += job_file_dir_content
-      job_files << step.files_dir if File.exists?(step.files_dir)
+      job_files << step.files_dir if File.exist?(step.files_dir)
       rec_dependencies = Set.new
 
       next unless recursive
@@ -99,7 +99,7 @@ class Step
         dep = Workflow.load_step path
         job_files << dep.path
         job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
-        job_files << dep.info_file if File.exists?(dep.info_file)
+        job_files << dep.info_file if File.exist?(dep.info_file)
       end
     end
 

data/lib/rbbt/workflow.rb

@@ -543,13 +543,13 @@ module Workflow
     persist = input_values.nil? ? false : true
     persist = false
 
-    if ! (Path === step_path ? step_path.find : File.exists?(step_path)) && step_path.split("/").length == 3 && File.exists?(new_path = Rbbt.var.jobs[step_path].find)
+    if ! (Path === step_path ? step_path.find : File.exist?(step_path)) && step_path.split("/").length == 3 && File.exist?(new_path = Rbbt.var.jobs[step_path].find)
       step_path = new_path
     end
     
     key = Path === step_path ? step_path.find : step_path
 
-    if ! File.exists?(step_path) && step_path.split("/").length == 3 && File.exists?(new_path = Rbbt.var.jobs[step_path].find)
+    if ! File.exist?(step_path) && step_path.split("/").length == 3 && File.exist?(new_path = Rbbt.var.jobs[step_path].find)
       step_path = new_path
     end
 
@@ -602,7 +602,7 @@ module Workflow
       pother = other.split(/\/+/)
       end_part = pother[-3..-1] * "/"
       new_path = prefix + "/" << end_part
-      if File.exists? new_path
+      if File.exist? new_path
         new_path 
       else
         Rbbt.var.jobs[end_part].find
@@ -615,7 +615,7 @@ module Workflow
     pother = other.split(/\/+/)
     end_part = pother[-3..-1] * "/"
     new_path = preal[0..-4] * "/" << "/" << end_part
-    return new_path if File.exists?(new_path) || File.exists?(new_path + '.info')
+    return new_path if File.exist?(new_path) || File.exist?(new_path + '.info')
     Rbbt.var.jobs[end_part].find
   end
 
@@ -713,7 +713,7 @@ module Workflow
   def self.load_step(path)
     path = Path.setup(path.dup) unless Path === path
 
-    if ! (Path === path ? path.exists? : File.exists?(path)) && path.split("/").length == 3 
+    if ! (Path === path ? path.exists? : File.exist?(path)) && path.split("/").length == 3 
       new_path = Rbbt.var.jobs[path]
       if new_path.exists? || new_path.set_extension('info').exists?
         path = new_path

data/share/rbbt_commands/hpc/clean

@@ -97,19 +97,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  different_system = job_batch_system != batch_system
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -126,17 +126,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   nodes = []
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   outt = File.mtime File.join(dir, 'std.out')
   errt = File.mtime File.join(dir, 'std.err')
   time_diff = Time.now - [outt, errt].max
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  aborted = error = true if aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
@@ -177,7 +177,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -189,7 +189,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/hpc/list

@@ -118,19 +118,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   container_home = nil
  end
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -147,13 +147,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    nodes = []
  end
 
- if File.exists?(File.join(dir, 'exit.status'))
+ if File.exist?(File.join(dir, 'exit.status'))
    now = File.ctime(File.join(dir, 'exit.status'))
  else
    now = Time.now
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   cerrt = File.ctime File.join(dir, 'std.err')
   coutt = File.ctime File.join(dir, 'std.out')
   outt = File.mtime File.join(dir, 'std.out')
@@ -163,10 +163,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  if done || error || aborted || running || queued || jobid 
    select = false
@@ -244,7 +244,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exist?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -316,7 +316,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/hpc/tail

@@ -46,7 +46,7 @@ if directory =~ /^[0-9]*$/
   end
 end
 
-raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
 
 require 'rbbt/hpc/slurm'
 

data/share/rbbt_commands/lsf/clean

@@ -97,19 +97,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  different_system = job_batch_system != batch_system
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -126,17 +126,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   nodes = []
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   outt = File.mtime File.join(dir, 'std.out')
   errt = File.mtime File.join(dir, 'std.err')
   time_diff = Time.now - [outt, errt].max
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  aborted = error = true if aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
@@ -177,7 +177,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -189,7 +189,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/lsf/list

@@ -118,19 +118,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   container_home = nil
  end
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -147,13 +147,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    nodes = []
  end
 
- if File.exists?(File.join(dir, 'exit.status'))
+ if File.exist?(File.join(dir, 'exit.status'))
    now = File.ctime(File.join(dir, 'exit.status'))
  else
    now = Time.now
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   cerrt = File.ctime File.join(dir, 'std.err')
   coutt = File.ctime File.join(dir, 'std.out')
   outt = File.mtime File.join(dir, 'std.out')
@@ -163,10 +163,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  if done || error || aborted || running || queued || jobid 
    select = false
@@ -244,7 +244,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exist?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -316,7 +316,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/lsf/tail

@@ -46,7 +46,7 @@ if directory =~ /^[0-9]*$/
   end
 end
 
-raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
 
 require 'rbbt/hpc/slurm'
 

data/share/rbbt_commands/slurm/clean

@@ -97,19 +97,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  different_system = job_batch_system != batch_system
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -126,17 +126,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   nodes = []
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   outt = File.mtime File.join(dir, 'std.out')
   errt = File.mtime File.join(dir, 'std.err')
   time_diff = Time.now - [outt, errt].max
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  aborted = error = true if aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
@@ -177,7 +177,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -189,7 +189,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    end
  end
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/slurm/list

@@ -118,19 +118,19 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   container_home = nil
  end
 
- if File.exists?(fid = File.join(dir, 'job.id'))
+ if File.exist?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
  else
   id = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'exit.status'))
+ if File.exist?(fstatus = File.join(dir, 'exit.status'))
   exit_status = Open.read(fstatus).to_i
  else
   exit_status = nil
  end
 
- if File.exists?(fstatus = File.join(dir, 'job.status'))
+ if File.exist?(fstatus = File.join(dir, 'job.status'))
    fstatus_txt = Open.read(fstatus) 
    begin
      if job_batch_system == "lsf"
@@ -147,13 +147,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    nodes = []
  end
 
- if File.exists?(File.join(dir, 'exit.status'))
+ if File.exist?(File.join(dir, 'exit.status'))
    now = File.ctime(File.join(dir, 'exit.status'))
  else
    now = Time.now
  end
 
- if File.exists?(File.join(dir, 'std.out'))
+ if File.exist?(File.join(dir, 'std.out'))
   cerrt = File.ctime File.join(dir, 'std.err')
   coutt = File.ctime File.join(dir, 'std.out')
   outt = File.mtime File.join(dir, 'std.out')
@@ -163,10 +163,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  fdep = File.join(dir, 'dependencies.list')
- deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+ deps = Open.read(fdep).split("\n") if File.exist?(fdep)
 
  fcadep = File.join(dir, 'canfail_dependencies.list')
- cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+ cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
  if done || error || aborted || running || queued || jobid 
    select = false
@@ -244,7 +244,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
  puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
+ puts Log.color(:magenta, "Output: ") << File.exist?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exist?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)") if long
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
@@ -316,7 +316,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
- if tail && File.exists?(File.join(dir, 'std.err'))
+ if tail && File.exist?(File.join(dir, 'std.err'))
    if exit_status && exit_status != 0
      puts Log.color(:magenta, "First error or exception found: ")
      puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read

data/share/rbbt_commands/slurm/tail

@@ -46,7 +46,7 @@ if directory =~ /^[0-9]*$/
   end
 end
 
-raise ParameterException, "Could not identify job #{directory}" unless File.exists?(directory)
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
 
 require 'rbbt/hpc/slurm'
 

data/share/rbbt_commands/tsv/select

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Select entries in a tsv based on some criteria
+
+$ rbbt tsv subset [options] file.tsv <key> <criteria>
+
+Subsets entries from a TSV file from a given list. Works with Tokyocabinet HDB and BDB as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-s--subset* Subset of keys (Comma-separated or file)
+-m--merge Merge TSV rows
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file, key, criteria = ARGV
+
+key, criteria, file = [file, key, nil] if criteria.nil?
+
+file = STDIN if file == '-' || file.nil?
+
+raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+
+options[:fields] = options[:fields].split(/,\|/) if options[:fields]
+options[:header_hash] = options["header_hash"]
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+  puts tsv.summary
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+  puts tsv.summary
+else
+  tsv = TSV.open(file, options)
+end
+
+criteria = criteria.to_regexp.to_regexp if criteria =~ /\/.*\/.*/
+puts tsv.select(key => criteria)

data/share/rbbt_commands/tsv/view

@@ -28,7 +28,7 @@ end
 
 file = ARGV.shift
 
-file = STDIN if file == '-'
+file = STDIN if file == '-' || file.nil?
 
 case
 when options[:tokyocabinet]
@@ -38,12 +38,12 @@ when options[:tokyocabinet_bd]
 else
   if String === file
     file = file.dup
-    Path.setup(File.expand_path(file))
+    file = Path.setup(File.expand_path(file))
   end
   tsv = file
 end
 
-tsv = TSV.open(tsv) if IO === tsv
+tsv = TSV.open(tsv) if IO === tsv || Path === tsv
 
 
 begin

data/share/rbbt_commands/workflow/archive_all

@@ -28,5 +28,5 @@ SOPT.usage if options[:help]
 
 target, *files = ARGV
 
-raise "Target already exists: #{ target }" if File.exists?(target) and not File.directory?(target)
+raise "Target already exists: #{ target }" if File.exist?(target) and not File.directory?(target)
 Step.archive(files, target)

data/share/rbbt_commands/workflow/prov

@@ -36,14 +36,14 @@ file = ARGV.shift
 
 $seen = []
 def get_step(file)
-  file = File.expand_path(file) if File.exists?(file)
+  file = File.expand_path(file) if File.exist?(file)
   file = file.sub(/\.(info|files)$/,'')
   $seen << file
   Workflow.load_step file
 end
 
 def touch(step)
-  return unless File.exists?(step.path)
+  return unless File.exist?(step.path)
   step.dependencies.each do |dep|
     next unless Open.exists?(dep.path)
     if Open.mtime(dep.path) > Open.mtime(step.path) + 1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.7
+  version: 5.34.10
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-26 00:00:00.000000000 Z
+date: 2022-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -452,6 +452,7 @@ files:
 - share/rbbt_commands/tsv/query
 - share/rbbt_commands/tsv/read
 - share/rbbt_commands/tsv/read_excel
+- share/rbbt_commands/tsv/select
 - share/rbbt_commands/tsv/slice
 - share/rbbt_commands/tsv/sort
 - share/rbbt_commands/tsv/subset