rbbt-util 5.29.0 → 5.30.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 169a7a3ce3ebf35313ca6f64d91e98bdb2bb7df79b0367f6888bf0978c67fb01
-  data.tar.gz: 654b5db6fea8f9ab2df8e8b00aaf945bdc9904965df0489ed72feba136ac6fd6
+  metadata.gz: 0275d0f8a0922abe3e0abee0f02935fb6e393452a68c0cad86de075b153a8457
+  data.tar.gz: 8a182bd87be2e1b4c538d0faaf8a16848e8f077c9f61f96f6df67ca07dd222f6
 SHA512:
-  metadata.gz: dd168909a2df9e5931f74ca9b91ee64e5ec6b003561d75d2843736321deb1e62ac5807133822be461d370b80ecc8b64885c9e20486548d949d20c90fe72cbbab
-  data.tar.gz: 377c2c070eb32856de9fb09107cc3ab72de64936590db2bb58ca2c4eaf057e954c258f1345d4b542bb39b2aba756f0c37bfdae999b9aacf01267a1bf327d13fa
+  metadata.gz: 02fd852c70355b89d187db04c52f27207012a2fe00f1fd7f414b95bea86bf47c3d19b0b766fa12d8520ee7c9270dfde369ce2a0c1dc60595e7a9b8062a176b92
+  data.tar.gz: 69591fffbf60d68a57dc319692adff4f4f28bb46d9b378fa23a0b01c55303b780d058f1e4ab1f41be52c43abb48ac8db7962e96f77fb2765f6f8f74587ee8825

data/lib/rbbt/hpc/orchestrate.rb

@@ -1,22 +1,110 @@
 require 'rbbt/workflow/util/orchestrator'
 module HPC
   module SLURM
-    def self.orchestrate_job(job, options, seen = {})
+
+    def self.job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
+
+      job_rules = IndiferentHash.setup(defaults.dup)
+
+      rules["chains"].each do |name,info|
+        IndiferentHash.setup(info)
+        chain_tasks = info[:tasks].split(/,\s*/)
+
+        chain_tasks.each do |task|
+          task_workflow, chain_task = task.split("#")
+          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
+          job_rules["chain_tasks"] ||= {}
+          job_rules["chain_tasks"][task_workflow] ||= []
+          job_rules["chain_tasks"][task_workflow]  << chain_task
+          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
+          config_keys = job_rules.delete :config_keys
+          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
+          if config_keys
+            config_keys.gsub!(/,\s+/,',') 
+            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+          end
+        end
+
+        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
+          break
+        else
+          job_rules.delete "chain_tasks" 
+        end
+      end if rules["chains"]
+
+      config_keys = job_rules.delete :config_keys
+      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
+
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+      end
+
+      if rules["skip"] && rules["skip"][workflow]
+        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
+      end
+
+      job_rules
+    end
+
+    def self.get_job_dependencies(job, job_rules)
+      deps = job.dependencies || []
+      deps += job.input_dependencies || []
+      deps
+    end
+
+    def self.orchestrate_job(job, options, skip = false, seen = {})
       return if job.done?
+      return unless job.path.split("/")[-4] == "jobs"
+      seen[:orchestration_target_job] ||= job
+
       options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
       options.delete "tail"
-      rules = YAML.load(Open.read(options[:rules])) if options[:rules]
+      options.delete "printfile"
+      options.delete "detach"
+
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
       rules ||= {}
+      IndiferentHash.setup(rules)
 
-      deps = job.dependencies || []
-      deps += job.input_dependencies || []
+      job_rules = self.job_rules(rules, job)
+
+      deps = get_job_dependencies(job, job_rules)
 
       dep_ids = deps.collect do |dep|
-        seen[dep.path] ||= self.orchestrate_job(dep, options.dup, seen)
-      end.compact 
+        skip_dep = job_rules["chain_tasks"] &&
+          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
+          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
+
+        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+        if job.canfail_paths.include? dep.path
+          [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
+        else
+          deps
+        end
+      end.flatten.compact.uniq
+
+      skip = true if job_rules[:skip]
+      return dep_ids if skip and seen[:orchestration_target_job] != job
+
+      job_rules.delete :chain_tasks
+      job_rules.delete :tasks
+      job_rules.delete :workflow
+      
+      config_keys = job_rules.delete(:config_keys)
+
+      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:slurm_dependencies => dep_ids))
+      job_options.delete :orchestration_rules
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
+      end
 
-      job_rules = Workflow::Orchestrator.job_rules(rules, job)
-      job_options = options.merge(job_rules).merge(:slurm_dependencies => dep_ids)
       run_job(job, job_options)
     end
   end

data/lib/rbbt/hpc/slurm.rb

@@ -33,7 +33,8 @@ module HPC
       group = File.basename(File.dirname(ENV['HOME']))
 
       if contain_and_sync
-        contain = "/scratch/tmp/rbbt-#{user}" if contain.nil?
+        random_file = TmpFile.random_name
+        contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
         sync = "~/.rbbt/var/jobs" if sync.nil?
         wipe_container = "post" if wipe_container.nil?
       end
@@ -58,11 +59,11 @@ module HPC
         when FalseClass 
           '--' << o << "=false"
         else
-          ['--' << o, "'#{v}'"] * " "
+          ['--' << o, "'#{v.to_s.gsub("'", '\'')}'"] * " "
         end
       end * " "
 
-      rbbt_cmd << " --config_keys='#{config_keys}'" if config_keys and not config_keys.empty?
+      rbbt_cmd << " --config_keys='#{config_keys.gsub("'", '\'')}'" if config_keys and not config_keys.empty?
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
@@ -76,6 +77,7 @@ module HPC
       fjob = File.join(slurm_basedir, 'job.id')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
+      fsyncexit = File.join(slurm_basedir, 'sync.status')
       fcmd = File.join(slurm_basedir, 'command.slurm')
 
       #{{{ GENERATE TEMPLATE
@@ -107,10 +109,6 @@ module HPC
         EOF
       end
 
-      header +=<<-EOF
-#CMD: #{rbbt_cmd}
-      EOF
-
       # ENV
       env = ""
       env +=<<-EOF
@@ -246,7 +244,7 @@ EOF
         end
 
         if contain
-          rbbt_cmd << " " << %(--workdir_all='#{contain}')
+          rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
         end
       end
 
@@ -254,16 +252,27 @@ EOF
       cmd =<<-EOF
 #{exec_cmd} \\
 #{rbbt_cmd}
+EOF
+      annotate_cmd =<<-EOF
+#{exec_cmd} \\
+workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID 
 EOF
 
+      header +=<<-EOF
+#CMD: #{rbbt_cmd}
+      EOF
+
       run +=<<-EOF
 
 # Run command
-#{cmd}
+step_path=$(#{cmd})
 
 # Save exit status
 exit_status=$?
 
+# Annotate info with SLURM job_info
+#{annotate_cmd}
+
 EOF
 
       # CODA
@@ -273,10 +282,10 @@ EOF
           coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean all -q &>> #{fsync}
 EOF
-        else
-          coda +=<<-EOF
-rbbt system clean all -q &>> #{fsync}
-EOF
+#        else
+#          coda +=<<-EOF
+#rbbt system clean all -q &>> #{fsync}
+#EOF
         end
 
         if sync.include?("=>")
@@ -285,7 +294,7 @@ EOF
           sync = sync.strip
           source = File.join(File.expand_path(contain), source)
         else
-          source = File.join(File.expand_path(contain), '.rbbt/var/jobs')
+          source = File.join(File.expand_path(contain), 'workdir/var/jobs')
         end
 
         target = File.expand_path(sync)
@@ -295,6 +304,7 @@ EOF
 mkdir -p "$(dirname '#{target}')"
 rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
 sync_es="$?" 
+echo $sync_es > #{fsyncexit}
 find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{source}", "#{target}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
 EOF
 
@@ -320,23 +330,24 @@ singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem
 EOF
           else
             coda +=<<-EOF
-#{exec_cmd} system clean
+##{exec_cmd} system clean
 if [ $exit_status == '0' -a $sync_es == '0' ]; then 
     rm -Rfv #{contain} &>> #{fsync}
 else
     echo "ERROR: Process failed or results could not sync correctly. Contain directory not purged" &>> #{fsync}
 fi
-unset sync_es
 EOF
 
           end
         end
       end
+
       coda +=<<-EOF
 
 # Write exit status to file
 echo $exit_status > #{fexit}
 EOF
+
       if sync
         coda +=<<-EOF 
 if [ "$sync_es" == '0' ]; then 
@@ -361,6 +372,11 @@ EOF
 
       slurm_basedir = options[:slurm_basedir]
       dependencies = options.delete :slurm_dependencies
+      dependencies = [] if dependencies.nil?
+
+      canfail_dependencies = dependencies.select{|dep| dep =~ /^canfail:(\d+)/ }.collect{|dep| dep.partition(":").last}
+      dependencies = dependencies.reject{|dep| dep =~ /^canfail:(\d+)/ }
+
       Open.mkdir slurm_basedir
 
       dry_run = options.delete :dry_run
@@ -369,6 +385,7 @@ EOF
       ferr  = File.join(slurm_basedir, 'std.err')
       fjob  = File.join(slurm_basedir, 'job.id')
       fdep  = File.join(slurm_basedir, 'dependencies.list')
+      fcfdep  = File.join(slurm_basedir, 'canfail_dependencies.list')
       fexit = File.join(slurm_basedir, 'exit.status')
       fsync = File.join(slurm_basedir, 'sync.log')
       fcmd  = File.join(slurm_basedir, 'command.slurm')
@@ -400,8 +417,21 @@ EOF
           Open.rm fexit
           Open.rm fout
           Open.rm ferr
+
           Open.write(fdep, dependencies * "\n") if dependencies.any?
-          dep_str = dependencies.any? ? "--dependency=afterok:" + dependencies * ":" : ''
+          Open.write(fcfdep, canfail_dependencies * "\n") if canfail_dependencies.any?
+
+
+          dep_str = '--dependency='
+          normal_dep_str = dependencies.any? ? "afterok:" + dependencies * ":" : nil
+          canfail_dep_str = canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+          if normal_dep_str.nil? && canfail_dep_str.nil?
+            dep_str = ""
+          else
+            dep_str += [normal_dep_str, canfail_dep_str].compact * ","
+          end
+
           job = CMD.cmd("sbatch #{dep_str} '#{fcmd}'").read.scan(/\d+/).first.to_i
           Log.debug "SBATCH job id: #{job}"
           Open.write(fjob, job.to_s)
@@ -494,6 +524,11 @@ EOF
       dry_run          = options.delete :dry_run
       tail             = options.delete :tail
       dependencies     = options.delete :slurm_dependencies
+      procpath         = options.delete :SLURM_procpath
+
+      options[:jobname] = job.clean_name
+      log_level         = options.delete :log
+      log_level       ||= Log.severity
 
       workflow = job.workflow
 
@@ -518,15 +553,13 @@ EOF
         inputs_dir = File.join(tmp_directory, 'inputs_dir')
         saved = Step.save_job_inputs(job, inputs_dir)
 
+        cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', log_level.to_s]
 
-        if saved && saved.any?
-          options[:inputs_dir] = inputs_dir
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
-        else
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--log', (options[:log] || Log.severity).to_s]
-        end
+        cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
+
+        cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
 
-        cmd << "--override_deps='#{override_deps}'" if override_deps and not override_deps.empty?
+        cmd << "--load_inputs='#{inputs_dir}'" if saved && saved.any?
 
         template = self.template(cmd, options)
         jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))

data/lib/rbbt/persist.rb

@@ -110,6 +110,8 @@ module Persist
   def self.load_file(path, type)
     begin
       case (type || :marshal).to_sym
+      when :path
+        path
       when :nil
         nil
       when :boolean
@@ -167,6 +169,8 @@ module Persist
     end
 
     case (type || :marshal).to_sym
+    when :path
+      nil
     when :nil
       nil
     when :boolean

data/lib/rbbt/persist/tsv/adapter.rb

@@ -104,9 +104,6 @@ module Persist
         write(true) if closed? || ! write?
         res = begin
                 yield
-              rescue Exception
-                Log.exception $!
-                raise $!
               ensure
                 close
               end
@@ -115,7 +112,6 @@ module Persist
     end
 
     def read_and_close
-      #return yield if @locked
       if read? || write?
         begin
           return yield
@@ -134,6 +130,41 @@ module Persist
       end
     end
 
+    def read_lock
+      read if closed?
+      if read?
+        return yield
+      end
+
+      lock do
+        close
+        read true
+        begin
+          yield
+        end
+      end
+    end
+
+    def write_lock
+      write if closed?
+      if write?
+        begin
+          return yield
+        ensure
+          close
+        end
+      end
+
+      lock do
+        close
+        write true
+        begin
+          yield
+        end
+      end
+    end
+
+
     def merge!(hash)
       hash.each do |key,values|
         self[key] = values
@@ -141,38 +172,38 @@ module Persist
     end
 
     def range(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def include?(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def [](*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def []=(*args)
-      self.write_and_close do
+      self.write_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def keys(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
 
     def prefix(key)
-      self.read_and_close do
+      self.read_lock do
         range(key, 1, key + MAX_CHAR, 1)
       end
     end
@@ -184,13 +215,13 @@ module Persist
 
 
     def size(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def each(*args, &block)
-      self.read_and_close do
+      self.read_lock do
         super(*args, &block)
       end
     end
@@ -208,7 +239,7 @@ module Persist
     end
 
     def values_at(*keys)
-      self.read_and_close do
+      self.read_lock do
         keys.collect do |k|
           self[k]
         end