rbbt-util 5.29.1 → 5.29.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b88fc549c1c1dc5cd56f06d933776d56540e3d0f4bacf77f04a449abcda974f
-  data.tar.gz: caf80ab624418c6c0a744038d98569c3a32d18dc8d9de162b42363e1c281e8c2
+  metadata.gz: fa648dbd0465a8c6a9a62820dc08a7f726dcd45cb541e7ef1c5e06b0e8343680
+  data.tar.gz: f05f659bfcfe3eef10a1d4ba786d40a7f835a08711ca23af9b5b619ddd61eeda
 SHA512:
-  metadata.gz: 8944f1d996afa5610f70046f4e61cf326461050f06f347529cd7e403440e0d9b47d0c33b862bc83e89d4479ce889e773aef0bbabfaf0dae797c3a2164f0e8a8c
-  data.tar.gz: fa0051835f35e23873bde6a075d81805d2878643e35e96e20c5d7423f0c7c48eaf25b2c5afad48215481b0ddd17b9d251555eae5b5c4a7fa420d9796dba9130b
+  metadata.gz: b44a34d7ad19eff67c7f8e6f3bfe0f55372e3d8a255247ff01d1aa858a904d2fcf73d45dd966702adf62bdbc3cb1b0dc3df643aaab747a922bfe3b747f975950
+  data.tar.gz: 5e74bcc37585452e22d876211f0c02c8fc431dbbdcc8cc0d3ef537ea794b722e0af9fc4ae743a014a1384c432f8f280761c74e35a1a8aaf25150067675a9f449

data/lib/rbbt/hpc/orchestrate.rb

@@ -1,23 +1,99 @@
 require 'rbbt/workflow/util/orchestrator'
 module HPC
   module SLURM
-    def self.orchestrate_job(job, options, seen = {})
+
+    def self.job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
+
+      job_rules = IndiferentHash.setup(defaults.dup)
+
+      rules["chains"].each do |name,info|
+        IndiferentHash.setup(info)
+        chain_tasks = info[:tasks].split(/,\s*/)
+
+        chain_tasks.each do |task|
+          task_workflow, chain_task = task.split("#")
+          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
+          job_rules["chain_tasks"] ||= {}
+          job_rules["chain_tasks"][task_workflow] ||= []
+          job_rules["chain_tasks"][task_workflow]  << chain_task
+          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
+          config_keys = job_rules.delete :config_keys
+          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
+          if config_keys
+            config_keys.gsub!(/,\s+/,',') 
+            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+          end
+        end
+
+        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
+          break
+        else
+          job_rules.delete "chain_tasks" 
+        end
+      end if rules["chains"]
+
+      config_keys = job_rules.delete :config_keys
+      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
+
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+      end
+
+      if rules["skip"] && rules["skip"][workflow]
+        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
+      end
+
+      job_rules
+    end
+
+    def self.get_job_dependencies(job, job_rules)
+      deps = job.dependencies || []
+      deps += job.input_dependencies || []
+      deps
+    end
+
+    def self.orchestrate_job(job, options, skip = false, seen = {})
       return if job.done?
       return unless job.path.split("/")[-4] == "jobs"
+      seen[:orchestration_target_job] ||= job
       options.delete "recursive_clean"
       options.delete "tail"
-      rules = YAML.load(Open.read(options[:rules])) if options[:rules]
+      options.delete "printfile"
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
       rules ||= {}
+      IndiferentHash.setup(rules)
 
-      deps = job.dependencies || []
-      deps += job.input_dependencies || []
+      job_rules = self.job_rules(rules, job)
+
+      deps = get_job_dependencies(job, job_rules)
 
       dep_ids = deps.collect do |dep|
-        seen[dep.path] ||= self.orchestrate_job(dep, options.dup, seen)
-      end.compact 
+        skip_dep = job_rules["chain_tasks"] &&
+          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
+          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
+        seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+      end.flatten.compact.uniq
+
+      skip = true if job_rules[:skip]
+      return dep_ids if skip and seen[:orchestration_target_job] != job
+
+      job_rules.delete :chain_tasks
+      job_rules.delete :tasks
+      job_rules.delete :workflow
+      
+      config_keys = job_rules.delete(:config_keys)
+
+      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:slurm_dependencies => dep_ids))
+      job_options.delete :orchestration_rules
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
+      end
 
-      job_rules = Workflow::Orchestrator.job_rules(rules, job)
-      job_options = options.merge(job_rules).merge(:slurm_dependencies => dep_ids)
       run_job(job, job_options)
     end
   end

data/lib/rbbt/hpc/slurm.rb

@@ -361,6 +361,7 @@ EOF
 
       slurm_basedir = options[:slurm_basedir]
       dependencies = options.delete :slurm_dependencies
+      dependencies = [] if dependencies.nil?
       Open.mkdir slurm_basedir
 
       dry_run = options.delete :dry_run

data/share/rbbt_commands/slurm/orchestrate

@@ -25,7 +25,7 @@ $slurm_options = SOPT.get <<EOF
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows
 -tm--time* Time
--R--rules* Orchestration rules
+-R--orchestration_rules* Orchestration rules
 -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.29.1
+  version: 5.29.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-14 00:00:00.000000000 Z
+date: 2021-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake