rbbt-util 5.32.26 → 5.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 072aa8b010c3f395051fd4e29f6ac22647944eec529bdf05be77ac6ad5260a37
-  data.tar.gz: d68843b2fe56019dd63165a34c97ec23d9e60933e4493bdb61f29985548bca69
+  metadata.gz: 1a6b198e57fe73c0e36e7aea6164bafd1a8a97d10b3a992f17d0ba6dc531a3c7
+  data.tar.gz: 5aa8b8753fc88d5de6fa8f6db31b9b71a57ee5dcc86380041582a4985984f749
 SHA512:
-  metadata.gz: 9d572a394b5736d8780021bc8ec65ba4381dc8cb84f111704854ce77ceca8da0ca72626e0144a04ab5c9b552a5566f4bdd553cad2a70050216fae231179beb95
-  data.tar.gz: ce51f8a4f0f5b4028148deb7fd3e57a4fbf5bb531f366378a2bae31354b799a66c6ba2768d9ae25578b3a0b09ee4e29f817f6ab768166bed39e4245b35a4a7e4
+  metadata.gz: a61b6e6671803d73b7993b2de9c6cf16d4f463ddd04f9378b54c77a00eb7292b4e817eea14cce81704fe3f2640d38ede202a3928d293ab1fe7e0e67fa87a35b0
+  data.tar.gz: d41b1477aab34bfc08610602a3e333a5701edb58d9fa10ecf24436659c4e996bb000ceccb20bbcba1f153ec4e888264ae47ab28689b9e2879601da0b7822e831

data/lib/rbbt/annotations/annotated_array.rb

@@ -94,6 +94,10 @@ module AnnotatedArray
     end
   end
 
+  def +(other)
+    self.annotate super(other)
+  end
+
 
   def reject
     res = []

data/lib/rbbt/annotations/util.rb

@@ -220,6 +220,35 @@ module Annotated
   def marshal_dump
     Annotated.purge(self).to_sym.to_s
   end
+
+  def self.to_hash(e)
+    hash = {}
+    if Array === e && AnntatedArray === e
+      hash[:literal] = Annotated.purge(e)
+      hash[:info] = e.info
+    elsif Array === e
+      hash = e.collect do |_e|
+        _hash = {}
+        _hash[:literal] = _e
+        _hash[:info] = _e.info
+        _hash
+      end
+    else
+      hash[:literal] = e
+      hash[:info] = e.info
+    end
+    hash
+  end
+
+  def self.load_hash(hash)
+    literal = hash[:literal]
+    info = hash[:info]
+    info[:annotation_types].each do |type|
+      type = Kernel.const_get(type) if String === type
+      type.setup(literal, info) 
+    end
+    literal
+  end
 end
 
 class String

data/lib/rbbt/entity.rb

@@ -164,7 +164,9 @@ module Entity
           define_method single_name, &block 
           define_method name do |*args|
             if Array === self
-              self.collect{|e| e.send(single_name, *args)}
+              res = self.collect{|e| e.send(single_name, *args)}
+              res.first.annotate(res) if Annotated === res.first && type == :single2array
+              res
             else
               self.send(single_name, *args)
             end

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -0,0 +1,152 @@
+require 'rbbt/hpc/orchestrate/rules'
+require 'rbbt/hpc/orchestrate/chains'
+
+module HPC
+  module Orchestration
+
+    def self.pb(batch)
+      if Array === batch
+        iii :BATCHES
+        batch.each{|b| pb b}
+        iii :END_BATCHES
+      else
+        n = batch.dup
+        n[:deps] = n[:deps].collect{|b| b[:top_level] }
+        iif n
+      end
+    end
+
+    def self.job_workload(job)
+      workload = []
+      path_jobs = {}
+
+      path_jobs[job.path] = job
+
+      heap = []
+      heap << job.path
+      while job_path = heap.pop
+        job = path_jobs[job_path]
+        next if job.done?
+        workload << job
+
+        deps =  job_dependencies(job)
+
+        deps.each do  |d|
+          path_jobs[d.path] ||= d
+        end
+
+        heap.concat deps.collect(&:path)
+        heap.uniq!
+      end
+      workload.uniq
+    end
+
+
+    def self.chain_batches(rules, chains, workload)
+      chain_rules = parse_chains(rules)
+
+      batches = []
+      while job = workload.pop
+        matches = chains.select{|name,info| info[:jobs].include? job }
+        if matches.any?
+          name, info = matches.sort_by do |name,info|
+            num_jobs = info[:jobs].length
+            total_tasks = chain_rules[name][:tasks].values.flatten.uniq.length
+            num_jobs.to_f + 1/total_tasks
+          end.last
+          workload = workload - info[:jobs]
+          info[:chain] = name
+          batch = info
+        else
+          batch = {:jobs => [job], :top_level => job}
+        end
+
+        chains.delete_if{|name,info| batch[:jobs].include? info[:top_level] }
+
+        chains.each do |name,info|
+          info[:jobs] = info[:jobs] - batch[:jobs]
+        end
+
+        chains.delete_if{|name,info| info[:jobs].length < 2 }
+
+        batches << batch
+      end
+
+      batches
+    end
+
+    def self.add_batch_deps(batches)
+
+      batches.each do |batch|
+        jobs = batch[:jobs]
+        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+        deps = all_deps.collect do |d|
+          (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
+        end.flatten.uniq
+        batch[:deps] = deps
+      end
+
+      batches
+    end
+
+    def self.add_rules_and_consolidate(rules, batches)
+      chain_rules = parse_chains(rules)
+
+      batches.each do |batch|
+        job_rules = batch[:jobs].inject(nil) do |acc,job|
+          task_rules = task_specific_rules(rules, job.workflow, job.task_name)
+          acc = accumulate_rules(acc, task_rules.dup)
+        end
+
+        if chain = batch[:chain]
+          batch[:rules] = merge_rules(chain_rules[chain][:rules].dup, job_rules)
+        else
+          batch[:rules] = job_rules
+        end
+      end
+
+      begin
+        batches.each do |batch|
+          batch[:deps] = batch[:deps].collect do |dep|
+            dep[:target] || dep
+          end if batch[:deps]
+        end
+
+        batches.each do |batch|
+          next unless batch[:rules][:skip]
+          batch[:rules].delete :skip
+          next if batch[:deps].nil?
+
+          if batch[:deps].any?
+            target = batch[:deps].select do |target|
+              (batch[:deps] - [target] - target[:deps]).empty?
+            end.first
+            next if target.nil?
+            target[:jobs] = batch[:jobs] + target[:jobs]
+            target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
+            target[:top_level] = batch[:top_level]
+            target[:rules] = accumulate_rules(target[:rules], batch[:rules])
+            batch[:target] = target
+          end
+          raise TryAgain
+        end
+      rescue TryAgain
+        retry
+      end
+
+      batches.delete_if{|b| b[:target] } 
+
+      batches
+    end
+
+    def self.job_batches(rules, job)
+      job_chains = self.job_chains(rules, job)
+
+      workload = job_workload(job).uniq
+
+      batches = chain_batches(rules, job_chains, workload)
+      batches = add_batch_deps(batches)
+      batches = add_rules_and_consolidate(rules, batches)
+    end
+  end
+end

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -0,0 +1,173 @@
+module HPC
+  module Orchestration
+    def self.check_chains(chains, job)
+      matches = []
+      chains.each do |name, chain|
+        next unless chain[:tasks].include?(job.workflow.to_s)
+        next unless chain[:tasks][job.workflow.to_s].include?(job.task_name.to_s)
+        matches << name
+      end
+      matches
+    end
+
+    def self.parse_chains(rules)
+      return {} if rules["chains"].nil?
+
+      chains = IndiferentHash.setup({})
+      rules["chains"].each do |name,rules|
+        rules  = IndiferentHash.setup(rules.dup)
+        chain_tasks = rules.delete(:tasks).split(/,\s*/)
+        workflow = rules.delete(:workflow)
+
+        chain_tasks.each do |task|
+          chain_workflow, chain_task = task.split("#")
+          chain_task, chain_workflow = chain_workflow, workflow if chain_task.nil? or chain_tasks.empty?
+
+          chains[name] ||= IndiferentHash.setup({:tasks => {}, :rules => rules })
+          chains[name][:tasks][chain_workflow] ||= []
+          chains[name][:tasks][chain_workflow] << chain_task
+        end
+      end
+
+      chains
+    end
+
+    def self.job_dependencies(job)
+      (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
+    end
+
+    #def self.job_workload(job)
+    #  workload = []
+    #  heap = []
+    #  heap << job
+    #  while job = heap.pop
+    #    next if job.done?
+    #    workload << job
+    #    heap.concat job_dependencies(job)
+    #    heap.uniq!
+    #  end
+    #  workload.uniq
+    #end
+
+    #def self.top_level_job(jobs)
+    #  top = jobs.select do |job|
+    #    (jobs - job_workload(job)).empty? &&
+    #      (job_workload(job) - jobs).select{|j| (job_workload(j) & jobs).any? }.empty?
+    #  end
+    #  return nil if top.length != 1
+    #  top.first
+    #end
+
+    #def self.job_chains(rules, job)
+    #  workload = job_workload(job)
+    #  chains = parse_chains(rules)
+
+    #  chain_jobs = {}
+    #  workload.each do |job|
+    #    check_chains(chains, job).each do |match|
+    #      chain_jobs[match] ||= []
+    #      chain_jobs[match] << job
+    #    end
+    #  end
+
+    #  job_chains = []
+
+    #  seen = []
+    #  chain_jobs.sort_by{|name,jobs| jobs.length }.reverse.each do |name,jobs|
+    #    remain = jobs - seen
+    #    next unless remain.length > 1
+    #    top_level_job = top_level_job(jobs)
+    #    next if top_level_job.nil?
+    #    job_chains << {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
+    #    seen.concat remain
+    #  end
+
+    #  job_chains
+    #end
+
+    #def self._job_chains(rules, job)
+    #  workload = job_workload(job)
+    #  chains = parse_chains(rules)
+
+    #  matches = check_chains(chains, job)
+
+    #  job_chains = {}
+    #  job.dependencies.each do |dep|
+    #    dep_chains = _job_chains(rules, dep)
+    #    matches.each do |match|
+    #      if dep_chains[match] && dep_chains[match].include?(dep)
+    #        dep_chains[match].prepend job
+    #      end
+    #    end
+    #    job_chains.merge!(dep_chains)
+    #  end
+
+    #  matches.each do |match|
+    #    job_chains[match] ||= [job]
+    #  end
+
+    #  job_chains
+    #end
+
+    #def self.job_chains(rules, job)
+    #  job_chains = self._job_chains(rules, job)
+    #  iif job_chains
+    #  chains = parse_chains(rules)
+
+    #  seen = []
+    #  job_chains.collect do |name,jobs|
+    #    remain = jobs - seen
+    #    next unless remain.length > 1
+    #    top_level_job = top_level_job(jobs)
+    #    next if top_level_job.nil?
+    #    seen.concat remain
+    #    {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
+    #  end.compact
+    #end
+
+    def self.job_chains(rules, job)
+      chains = self.parse_chains(rules)
+
+      matches = check_chains(chains, job)
+
+      dependencies = job_dependencies(job)
+
+      job_chains = []
+      new_job_chains = {}
+      dependencies.each do |dep|
+        dep_matches = check_chains(chains, dep)
+        common = matches & dep_matches
+
+        dep_chains = job_chains(rules, dep)
+        found = []
+        dep_chains.each do |match,info|
+          if common.include?(match)
+            found << match
+            new_info = new_job_chains[match] ||= {}
+            new_info[:jobs] ||= []
+            new_info[:jobs].concat info[:jobs]
+            new_info[:top_level] = job
+          else
+            job_chains << [match, info]
+          end
+        end
+
+        (common - found).each do |match|
+          info = {}
+          info[:jobs] = [job, dep]
+          info[:top_level] = job
+          job_chains << [match, info]
+        end
+      end
+
+      new_job_chains.each do |match,info|
+        info[:jobs].prepend job
+        job_chains << [match, info]
+      end
+
+      job_chains
+    end
+
+  end
+end
+

data/lib/rbbt/hpc/orchestrate/rules.rb

@@ -0,0 +1,70 @@
+module HPC
+  module Orchestration
+    def self.add_config_keys(current, new)
+      if current.nil?
+        new
+      else
+        new + ',' + current
+      end.gsub(/,\s*/,',').split(",").reverse.uniq.reverse * ","
+    end
+
+    def self.workflow_rules(rules, workflow)
+      return {} if rules[workflow].nil?
+      return {} if rules[workflow]["defaults"].nil? 
+      IndiferentHash.setup(rules[workflow]["defaults"])
+    end
+
+    def self.merge_rules(current, new)
+      return IndiferentHash.setup({}) if (new.nil? || new.empty?) && (current.nil? || current.empty?)
+      return IndiferentHash.setup(current.dup) if new.nil? || new.empty?
+      return IndiferentHash.setup(new.dup) if current.nil? || current.empty?
+      target = IndiferentHash.setup(current.dup)
+      new.each do |k,value|
+        case k.to_s
+        when "config_keys"
+          target[k] = add_config_keys target["config_keys"], value
+        else
+          next if target.include?(k)
+          target[k] = value
+        end
+      end
+      target
+    end
+
+    def self.accumulate_rules(current, new)
+      return IndiferentHash.setup({}) if (new.nil? || new.empty?) && (current.nil? || current.empty?)
+      return IndiferentHash.setup(current.dup) if new.nil? || new.empty?
+      return IndiferentHash.setup(new.dup) if current.nil? || current.empty?
+      target = IndiferentHash.setup(current.dup)
+      new.each do |k,value|
+        case k.to_s
+        when "config_keys"
+          target[k] = add_config_keys target["config_keys"], value
+        when "cpus"
+          target[k] = [target[k], value].compact.sort_by{|v| v.to_i}.last
+        when "time"
+          target[k] = Misc.format_seconds [target[k], value].compact.inject(0){|acc,t|  acc += Misc.timespan t }
+        when "skip"
+          skip = target[k] && value
+          target.delete k unless skip
+        else
+          next if target.include?(k)
+          target[k] = value
+        end
+      end
+      target
+    end
+
+    def self.task_specific_rules(rules, workflow, task)
+      defaults = rules[:defaults] || {}
+      workflow = workflow.to_s
+      task = task.to_s
+      return defaults if rules[workflow].nil?
+      workflow_rules = merge_rules(workflow_rules(rules, workflow), defaults)
+      return IndiferentHash.setup(workflow_rules.dup) if rules[workflow][task].nil?
+      merge_rules(rules[workflow][task], workflow_rules)
+    end
+
+
+  end
+end

data/lib/rbbt/hpc/orchestrate.old.rb

@@ -0,0 +1,220 @@
+require 'rbbt/workflow/util/orchestrator'
+module HPC
+  module Orchestration
+    def job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+      task_name = job.overriden.to_s if Symbol === job.overriden
+
+      defaults = rules["defaults"] || {}
+      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
+
+      job_rules = IndiferentHash.setup(defaults.dup)
+
+      rules["chains"].each do |name,info|
+        IndiferentHash.setup(info)
+        chain_tasks = info[:tasks].split(/,\s*/)
+
+        chain_tasks.each do |task|
+          task_workflow, chain_task = task.split("#")
+          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
+          job_rules["chain_tasks"] ||= {}
+          job_rules["chain_tasks"][task_workflow] ||= []
+          job_rules["chain_tasks"][task_workflow]  << chain_task
+          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
+          config_keys = job_rules.delete :config_keys
+          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
+          if config_keys
+            config_keys.gsub!(/,\s+/,',') 
+            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+          end
+        end
+
+        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
+          break
+        else
+          job_rules.delete "chain_tasks" 
+        end
+      end if rules["chains"]
+
+      config_keys = job_rules.delete :config_keys
+      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
+
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+      end
+
+      if rules["skip"] && rules["skip"][workflow]
+        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
+      end
+
+      job_rules
+    end
+
+    def get_job_dependencies(job, job_rules = nil)
+      deps = job.dependencies || []
+      deps += job.input_dependencies || []
+      deps
+    end
+
+    def get_recursive_job_dependencies(job)
+      deps = get_job_dependencies(job) 
+      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
+    end
+
+    def piggyback(job, job_rules, job_deps)
+      return false unless job_rules["skip"]
+      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
+      final_deps = final_deps.reject{|dep| dep.done? }
+      return final_deps.first if final_deps.length == 1
+      return false
+    end
+
+    def get_chains(job, rules, chains = {})
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      input_deps = []
+      job.rec_dependencies.each do |dep|
+        input_deps.concat dep.input_dependencies
+      end
+
+      job_deps.each do |dep|
+        input_deps.concat dep.input_dependencies
+        get_chains(dep, rules, chains)
+      end
+
+      job_deps.select do |dep|
+        chained = job_rules["chain_tasks"] &&
+          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
+          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
+
+        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
+        chained || dep_skip
+      end.each do |dep|
+        chains[job] ||= [] 
+        chains[job] << dep 
+        chains[job].concat chains[dep] if chains[dep]
+        chains[job].uniq!
+      end
+
+      chains
+    end
+
+    def workload(job, rules, chains, options, seen = nil)
+      return [] if job.done?
+      if seen.nil?
+        seen = {}
+        target_job = true
+      end
+
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      chain = chains[job]
+      chain = chain.reject{|j| seen.include? j.path} if chain
+      chain = chain.reject{|dep| dep.done? } if chain
+      piggyback = piggyback(job, job_rules, job_deps)
+      dep_ids = job_deps.collect do |dep|
+        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
+        next_options = IndiferentHash.setup(options.dup)
+        if piggyback and piggyback == dep
+          next_options[:piggyback] ||= []
+          next_options[:piggyback].push job
+          ids = workload(dep, rules, chains, next_options, seen)
+        else
+          next_options.delete :piggyback
+          ids = workload(dep, rules, chains, next_options, seen)
+        end
+
+        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
+
+        seen[dep.path] = ids
+        ids
+      end.compact.flatten.uniq
+
+      return seen[job.path] || dep_ids if seen.include?(job.path)
+
+      if piggyback and seen[piggyback.path]
+        return seen[job.path] = seen[piggyback.path] 
+      end
+
+      job_rules.delete :chain_tasks
+      job_rules.delete :tasks
+      job_rules.delete :workflow
+      
+
+      option_config_keys = options[:config_keys]
+
+      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
+      job_options.delete :orchestration_rules
+
+      config_keys = job_rules.delete(:config_keys)
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
+      end
+
+      if option_config_keys
+        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
+      end
+
+      if options[:piggyback]
+        manifest = options[:piggyback].uniq
+        manifest += [job]
+        manifest.concat chain if chain
+
+        job = options[:piggyback].first
+
+        job_rules = self.job_rules(rules, job)
+        new_config_keys = self.job_rules(rules, job)[:config_keys]
+        if new_config_keys
+          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
+          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
+        end
+
+        job_options.delete :piggyback
+      else
+        manifest = [job]
+        manifest.concat chain if chain
+      end
+
+      manifest.uniq!
+
+      job_options[:manifest] = manifest.collect{|j| j.task_signature }
+
+      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
+
+      if options[:dry_run]
+        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+        job_options[:manifest].first
+      else
+        run_job(job, job_options)
+      end
+    end
+
+
+    def orchestrate_job(job, options)
+      options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
+      options.delete "tail"
+      options.delete "printpath"
+      options.delete "detach"
+      options.delete "jobname"
+
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules ||= {}
+      IndiferentHash.setup(rules)
+
+      chains = get_chains(job, rules)
+
+      workload(job, rules, chains, options)
+    end
+
+  end
+end
+