rbbt-util 5.29.2 → 5.30.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa648dbd0465a8c6a9a62820dc08a7f726dcd45cb541e7ef1c5e06b0e8343680
-  data.tar.gz: f05f659bfcfe3eef10a1d4ba786d40a7f835a08711ca23af9b5b619ddd61eeda
+  metadata.gz: 86a6fb4f62fc52b64a090ff3068452055f79ec65f5d11a1a497d1433370b76e2
+  data.tar.gz: 453409b866e291e8971d13ef737ac8b6666ba89251b22a756bebc37339fa4ea7
 SHA512:
-  metadata.gz: b44a34d7ad19eff67c7f8e6f3bfe0f55372e3d8a255247ff01d1aa858a904d2fcf73d45dd966702adf62bdbc3cb1b0dc3df643aaab747a922bfe3b747f975950
-  data.tar.gz: 5e74bcc37585452e22d876211f0c02c8fc431dbbdcc8cc0d3ef537ea794b722e0af9fc4ae743a014a1384c432f8f280761c74e35a1a8aaf25150067675a9f449
+  metadata.gz: 8438fa94a7f460192656be9bd300b2b9991fd5b97d68cdfdf3d33baa76ed8f80c53caaa967e0ecd317cb6d99a32ca0e0fede252b58c96d913b5b44bbdcdfb8f0
+  data.tar.gz: f6d50c0b5aba669526c0d044a15598a07ded038b3ae7964c84723de05c93e30b30763a5ceddbd300a9c5f79f1c84b9305ebaa786c67702de32c5a0b26ce56222

data/lib/rbbt/hpc/orchestrate.rb

@@ -60,9 +60,14 @@ module HPC
       return if job.done?
       return unless job.path.split("/")[-4] == "jobs"
       seen[:orchestration_target_job] ||= job
+
       options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
       options.delete "tail"
       options.delete "printfile"
+      options.delete "detach"
+
       rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
       rules ||= {}
       IndiferentHash.setup(rules)
@@ -75,7 +80,13 @@ module HPC
         skip_dep = job_rules["chain_tasks"] &&
           job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
           job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
-        seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+
+        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+        if job.canfail_paths.include? dep.path
+          [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
+        else
+          deps
+        end
       end.flatten.compact.uniq
 
       skip = true if job_rules[:skip]

data/lib/rbbt/hpc/slurm.rb

@@ -33,7 +33,8 @@ module HPC
       group = File.basename(File.dirname(ENV['HOME']))
 
       if contain_and_sync
-        contain = "/scratch/tmp/rbbt-#{user}" if contain.nil?
+        random_file = TmpFile.random_name
+        contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
         sync = "~/.rbbt/var/jobs" if sync.nil?
         wipe_container = "post" if wipe_container.nil?
       end
@@ -58,11 +59,11 @@ module HPC
         when FalseClass 
           '--' << o << "=false"
         else
-          ['--' << o, "'#{v}'"] * " "
+          ['--' << o, "'#{v.to_s.gsub("'", '\'')}'"] * " "
         end
       end * " "
 
-      rbbt_cmd << " --config_keys='#{config_keys}'" if config_keys and not config_keys.empty?
+      rbbt_cmd << " --config_keys='#{config_keys.gsub("'", '\'')}'" if config_keys and not config_keys.empty?
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
@@ -76,6 +77,7 @@ module HPC
       fjob = File.join(slurm_basedir, 'job.id')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
+      fsyncexit = File.join(slurm_basedir, 'sync.status')
       fcmd = File.join(slurm_basedir, 'command.slurm')
 
       #{{{ GENERATE TEMPLATE
@@ -107,10 +109,6 @@ module HPC
         EOF
       end
 
-      header +=<<-EOF
-#CMD: #{rbbt_cmd}
-      EOF
-
       # ENV
       env = ""
       env +=<<-EOF
@@ -246,7 +244,7 @@ EOF
         end
 
         if contain
-          rbbt_cmd << " " << %(--workdir_all='#{contain}')
+          rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
         end
       end
 
@@ -254,16 +252,27 @@ EOF
       cmd =<<-EOF
 #{exec_cmd} \\
 #{rbbt_cmd}
+EOF
+      annotate_cmd =<<-EOF
+#{exec_cmd} \\
+workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID 
 EOF
 
+      header +=<<-EOF
+#CMD: #{rbbt_cmd}
+      EOF
+
       run +=<<-EOF
 
 # Run command
-#{cmd}
+step_path=$(#{cmd})
 
 # Save exit status
 exit_status=$?
 
+# Annotate info with SLURM job_info
+#{annotate_cmd}
+
 EOF
 
       # CODA
@@ -273,10 +282,10 @@ EOF
           coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean all -q &>> #{fsync}
 EOF
-        else
-          coda +=<<-EOF
-rbbt system clean all -q &>> #{fsync}
-EOF
+#        else
+#          coda +=<<-EOF
+#rbbt system clean all -q &>> #{fsync}
+#EOF
         end
 
         if sync.include?("=>")
@@ -285,7 +294,7 @@ EOF
           sync = sync.strip
           source = File.join(File.expand_path(contain), source)
         else
-          source = File.join(File.expand_path(contain), '.rbbt/var/jobs')
+          source = File.join(File.expand_path(contain), 'workdir/var/jobs')
         end
 
         target = File.expand_path(sync)
@@ -295,6 +304,7 @@ EOF
 mkdir -p "$(dirname '#{target}')"
 rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
 sync_es="$?" 
+echo $sync_es > #{fsyncexit}
 find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{source}", "#{target}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
 EOF
 
@@ -320,23 +330,24 @@ singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem
 EOF
           else
             coda +=<<-EOF
-#{exec_cmd} system clean
+##{exec_cmd} system clean
 if [ $exit_status == '0' -a $sync_es == '0' ]; then 
     rm -Rfv #{contain} &>> #{fsync}
 else
     echo "ERROR: Process failed or results could not sync correctly. Contain directory not purged" &>> #{fsync}
 fi
-unset sync_es
 EOF
 
           end
         end
       end
+
       coda +=<<-EOF
 
 # Write exit status to file
 echo $exit_status > #{fexit}
 EOF
+
       if sync
         coda +=<<-EOF 
 if [ "$sync_es" == '0' ]; then 
@@ -362,6 +373,10 @@ EOF
       slurm_basedir = options[:slurm_basedir]
       dependencies = options.delete :slurm_dependencies
       dependencies = [] if dependencies.nil?
+
+      canfail_dependencies = dependencies.select{|dep| dep =~ /^canfail:(\d+)/ }.collect{|dep| dep.partition(":").last}
+      dependencies = dependencies.reject{|dep| dep =~ /^canfail:(\d+)/ }
+
       Open.mkdir slurm_basedir
 
       dry_run = options.delete :dry_run
@@ -370,6 +385,7 @@ EOF
       ferr  = File.join(slurm_basedir, 'std.err')
       fjob  = File.join(slurm_basedir, 'job.id')
       fdep  = File.join(slurm_basedir, 'dependencies.list')
+      fcfdep  = File.join(slurm_basedir, 'canfail_dependencies.list')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
       fcmd  = File.join(slurm_basedir, 'command.slurm')
@@ -401,8 +417,21 @@ EOF
           Open.rm fexit
           Open.rm fout
           Open.rm ferr
+
           Open.write(fdep, dependencies * "\n") if dependencies.any?
-          dep_str = dependencies.any? ? "--dependency=afterok:" + dependencies * ":" : ''
+          Open.write(fcfdep, canfail_dependencies * "\n") if canfail_dependencies.any?
+
+
+          dep_str = '--dependency='
+          normal_dep_str = dependencies.any? ? "afterok:" + dependencies * ":" : nil
+          canfail_dep_str = canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+          if normal_dep_str.nil? && canfail_dep_str.nil?
+            dep_str = ""
+          else
+            dep_str += [normal_dep_str, canfail_dep_str].compact * ","
+          end
+
           job = CMD.cmd("sbatch #{dep_str} '#{fcmd}'").read.scan(/\d+/).first.to_i
           Log.debug "SBATCH job id: #{job}"
           Open.write(fjob, job.to_s)
@@ -495,7 +524,11 @@ EOF
       dry_run          = options.delete :dry_run
       tail             = options.delete :tail
       dependencies     = options.delete :slurm_dependencies
+      procpath         = options.delete :SLURM_procpath
+
       options[:jobname] = job.clean_name
+      log_level         = options.delete :log
+      log_level       ||= Log.severity
 
       workflow = job.workflow
 
@@ -520,14 +553,13 @@ EOF
         inputs_dir = File.join(tmp_directory, 'inputs_dir')
         saved = Step.save_job_inputs(job, inputs_dir)
 
-        if saved && saved.any?
-          options[:inputs_dir] = inputs_dir
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
-        else
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--log', (options[:log] || Log.severity).to_s]
-        end
+        cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', log_level.to_s]
+
+        cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
+
+        cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
 
-        cmd << "--override_deps='#{override_deps}'" if override_deps and not override_deps.empty?
+        cmd << "--load_inputs='#{inputs_dir}'" if saved && saved.any?
 
         template = self.template(cmd, options)
         jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))

data/lib/rbbt/persist.rb

@@ -110,6 +110,8 @@ module Persist
   def self.load_file(path, type)
     begin
       case (type || :marshal).to_sym
+      when :path
+        path
       when :nil
         nil
       when :boolean
@@ -167,6 +169,8 @@ module Persist
     end
 
     case (type || :marshal).to_sym
+    when :path
+      nil
     when :nil
       nil
     when :boolean

data/lib/rbbt/persist/tsv/adapter.rb

@@ -104,9 +104,6 @@ module Persist
         write(true) if closed? || ! write?
         res = begin
                 yield
-              rescue Exception
-                Log.exception $!
-                raise $!
               ensure
                 close
               end
@@ -115,7 +112,6 @@ module Persist
     end
 
     def read_and_close
-      #return yield if @locked
       if read? || write?
         begin
           return yield
@@ -134,6 +130,41 @@ module Persist
       end
     end
 
+    def read_lock
+      read if closed?
+      if read?
+        return yield
+      end
+
+      lock do
+        close
+        read true
+        begin
+          yield
+        end
+      end
+    end
+
+    def write_lock
+      write if closed?
+      if write?
+        begin
+          return yield
+        ensure
+          close
+        end
+      end
+
+      lock do
+        close
+        write true
+        begin
+          yield
+        end
+      end
+    end
+
+
     def merge!(hash)
       hash.each do |key,values|
         self[key] = values
@@ -141,38 +172,38 @@ module Persist
     end
 
     def range(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def include?(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def [](*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def []=(*args)
-      self.write_and_close do
+      self.write_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def keys(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
 
     def prefix(key)
-      self.read_and_close do
+      self.read_lock do
         range(key, 1, key + MAX_CHAR, 1)
       end
     end
@@ -184,13 +215,13 @@ module Persist
 
 
     def size(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def each(*args, &block)
-      self.read_and_close do
+      self.read_lock do
         super(*args, &block)
       end
     end
@@ -208,7 +239,7 @@ module Persist
     end
 
     def values_at(*keys)
-      self.read_and_close do
+      self.read_lock do
         keys.collect do |k|
           self[k]
         end

data/lib/rbbt/tsv.rb

@@ -113,11 +113,12 @@ module TSV
 
     data.entity_options = entity_options
 
-    if Path === source and data.identifiers
-      data.identifiers = Path.setup(data.identifiers, source.pkgdir, source.resource)
+    if Path === source && data.identifiers
+      Path.setup(data.identifiers, source.pkgdir, source.resource)
     end
 
     if data.respond_to? :persistence_path
+      data.read
       data
     else
       h = data.dup

data/lib/rbbt/util/cmd.rb

@@ -217,7 +217,7 @@ module CMD
     end
   end
 
-  def self.cmd_log(*args)
+  def self.cmd_pid(*args)
     all_args = *args
 
     all_args << {} unless Hash === all_args.last
@@ -248,4 +248,9 @@ module CMD
     nil
   end
 
+  def self.cmd_log(*args)
+    cmd_pid(*args)
+    nil
+  end
+
 end

data/lib/rbbt/util/misc/options.rb

@@ -242,48 +242,6 @@ module Misc
 
     return options
 
-    options = {}
-    string.split(/#/).each do |str|
-      if str.match(/(.*)=(.*)/)
-        option, value = $1, $2
-      else
-        option, value = str, true
-      end
-
-      option = option.sub(":",'').to_sym if option.chars.first == ':'
-      value  = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
-
-      if value == true
-        options[option] = option.to_s.chars.first != '!' 
-      else
-        options[option] = Thread.start do
-          $SAFE = 0;
-          case 
-          when value =~ /^(?:true|T)$/i
-            true
-          when value =~ /^(?:false|F)$/i
-            false
-          when Symbol === value
-            value
-          when (String === value and value =~ /^\/(.*)\/$/)
-            Regexp.new /#{$1}/
-          else
-            begin
-              Kernel.const_get value
-            rescue
-              begin  
-                raise if value =~ /[a-z]/ and defined? value
-                eval(value) 
-              rescue Exception
-                value 
-              end
-            end
-          end
-        end.value
-      end
-    end
-
-    options
   end
 
 end