rbbt-util 5.32.24 → 5.32.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a7eebab331a8908a290e4a3ad8db2ee3dc3c3dec07b93ec1c74f504f9f7ea29
-  data.tar.gz: 0f9a2ce6019676118ab439dea57f16605555d9ef850c35998704b9215c477ffb
+  metadata.gz: a680ce3f6d91a1b0ff8060ee18b8e68efbd43df56f238869b30f6f88617d23d9
+  data.tar.gz: d6c8c724f15d4ce4c910ff9db6becbd0587c8dc04598abcc934884a8368dd93e
 SHA512:
-  metadata.gz: 2d81532c2babaae631ca98784e7b7e36039fed68544b6bce9fc8fc2d74a155ffd98fe0752477c7ad245aaa0c65e82c47f330f65c5d70a7f5f20a77de4bc4dbd6
-  data.tar.gz: 2914786ec23349cabccbb6e8a1c561336b60dd5d862e89c3a95802f040c66f8107d5a2135455f00d8bc7b9e06e390fd6462a861419b78ae25bf3a58c14a33350
+  metadata.gz: 9b90c640f51de96d469af4b442d9ba66aa89a592c7776196058da3b20e13b07ef50f67e07018435b12bed13492c3bf9d9abf4de565902094a3d58be4ab628fd5
+  data.tar.gz: 523d7234c59bc1781d47827daebb07870913b8dec9851110d5c2e0c92441903601965f9a7bb4b494b0271d64e09ad0f3b80b254176ee3495744c2bbf1f34f714

data/bin/rbbt_find.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find a path
+
+$ #{$0} [options] [<subpath>] <path>
+
+Use - to read from STDIN
+
+-h--help Print this help
+-w--workflows Workflow to load
+-s--search_path* Workflow to load
+-l--list List contents of resolved directories
+-n--nocolor Don't color output
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+subpath, path = ARGV
+path, subpath = subpath, nil if path.nil?
+
+begin
+  require 'rbbt/workflow'
+  workflow = Workflow.require_workflow subpath
+  subpath = workflow.libdir
+rescue
+  Log.exception $!
+end if subpath && subpath =~ /^[A-Z][a-zA-Z]+$/
+
+path = subpath ? Path.setup(subpath)[path] : Path.setup(path)
+
+search_path = options[:search_path].to_sym if options.include? :search_path
+nocolor = options[:nocolor]
+
+found = if search_path
+          [path.find(search_path)]
+        else
+          path.find_all
+        end
+
+found.each do |path|
+  if options[:list] && File.directory?(path)
+    puts Log.color :blue, path
+    path.glob("*").each do |subpath|
+      if nocolor
+        puts subpath
+      else
+        color = File.directory?(subpath) ? :blue : nil
+        puts " " << Log.color(color, subpath)
+      end
+    end
+  else
+    if nocolor
+      puts path
+    else
+      color = File.exists?(path) ? (File.directory?(path) ? :blue : nil) : :red
+      puts Log.color color, path
+    end
+
+  end
+end
+

data/lib/rbbt/annotations/annotated_array.rb

@@ -94,6 +94,10 @@ module AnnotatedArray
     end
   end
 
+  def +(other)
+    self.annotate super(other)
+  end
+
 
   def reject
     res = []

data/lib/rbbt/annotations/util.rb

@@ -220,6 +220,35 @@ module Annotated
   def marshal_dump
     Annotated.purge(self).to_sym.to_s
   end
+
+  def self.to_hash(e)
+    hash = {}
+    if Array === e && AnntatedArray === e
+      hash[:literal] = Annotated.purge(e)
+      hash[:info] = e.info
+    elsif Array === e
+      hash = e.collect do |_e|
+        _hash = {}
+        _hash[:literal] = _e
+        _hash[:info] = _e.info
+        _hash
+      end
+    else
+      hash[:literal] = e
+      hash[:info] = e.info
+    end
+    hash
+  end
+
+  def self.load_hash(hash)
+    literal = hash[:literal]
+    info = hash[:info]
+    info[:annotation_types].each do |type|
+      type = Kernel.const_get(type) if String === type
+      type.setup(literal, info) 
+    end
+    literal
+  end
 end
 
 class String

data/lib/rbbt/entity.rb

@@ -164,7 +164,9 @@ module Entity
           define_method single_name, &block 
           define_method name do |*args|
             if Array === self
-              self.collect{|e| e.send(single_name, *args)}
+              res = self.collect{|e| e.send(single_name, *args)}
+              res.first.annotate(res) if Annotated === res.first && type == :single2array
+              res
             else
               self.send(single_name, *args)
             end

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -0,0 +1,140 @@
+require 'rbbt/hpc/orchestrate/rules'
+require 'rbbt/hpc/orchestrate/chains'
+
+module HPC
+  module Orchestration
+
+    def self.pb(batch)
+      if Array === batch
+        iii :BATCHES
+        batch.each{|b| pb b}
+        iii :END_BATCHES
+      else
+        n = batch.dup
+        n[:deps] = n[:deps].collect{|b| b[:top_level] }
+        iif n
+      end
+    end
+
+    def self.job_workload(job)
+      workload = []
+      heap = []
+      heap << job
+      while job = heap.pop
+        next if job.done?
+        workload << job
+        heap.concat job_dependencies(job)
+        heap.uniq!
+      end
+      workload.uniq
+    end
+
+
+    def self.chain_batches(rules, chains, workload)
+      chain_rules = parse_chains(rules)
+
+      batches = []
+      while job = workload.pop
+        matches = chains.select{|name,info| info[:jobs].include? job }
+        if matches.any?
+          name, info = matches.sort_by do |name,info|
+            num_jobs = info[:jobs].length
+            total_tasks = chain_rules[name][:tasks].values.flatten.uniq.length
+            num_jobs.to_f + 1/total_tasks
+          end.last
+          workload = workload - info[:jobs]
+          info[:chain] = name
+          batch = info
+        else
+          batch = {:jobs => [job], :top_level => job}
+        end
+
+        chains.delete_if{|name,info| batch[:jobs].include? info[:top_level] }
+
+        chains.each do |name,info|
+          info[:jobs] = info[:jobs] - batch[:jobs]
+        end
+
+        chains.delete_if{|name,info| info[:jobs].length < 2 }
+
+        batches << batch
+      end
+
+      batches
+    end
+
+    def self.add_batch_deps(batches)
+
+      batches.each do |batch|
+        jobs = batch[:jobs]
+        all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
+        deps = all_deps.collect do |d|
+          (batches - [batch]).select{|batch| batch[:jobs].include? d }
+        end.flatten.uniq
+        batch[:deps] = deps
+      end
+
+      batches
+    end
+
+    def self.add_rules_and_consolidate(rules, batches)
+      chain_rules = parse_chains(rules)
+
+      batches.each do |batch|
+        job_rules = batch[:jobs].inject(nil) do |acc,job|
+          task_rules = task_specific_rules(rules, job.workflow, job.task_name)
+          acc = accumulate_rules(acc, task_rules.dup)
+        end
+
+        if chain = batch[:chain]
+          batch[:rules] = merge_rules(chain_rules[chain][:rules].dup, job_rules)
+        else
+          batch[:rules] = job_rules
+        end
+      end
+
+      begin
+        batches.each do |batch|
+          batch[:deps] = batch[:deps].collect do |dep|
+            dep[:target] || dep
+          end if batch[:deps]
+        end
+
+        batches.each do |batch|
+          next unless batch[:rules][:skip]
+          batch[:rules].delete :skip
+          next if batch[:deps].nil?
+
+          if batch[:deps].any?
+            target = batch[:deps].select do |target|
+              (batch[:deps] - [target] - target[:deps]).empty?
+            end.first
+            next if target.nil?
+            target[:jobs] = batch[:jobs] + target[:jobs]
+            target[:deps] = (target[:deps] + batch[:deps]).uniq - [target]
+            target[:top_level] = batch[:top_level]
+            target[:rules] = accumulate_rules(target[:rules], batch[:rules])
+            batch[:target] = target
+          end
+          raise TryAgain
+        end
+      rescue TryAgain
+        retry
+      end
+
+      batches.delete_if{|b| b[:target] } 
+
+      batches
+    end
+
+    def self.job_batches(rules, job)
+      job_chains = self.job_chains(rules, job)
+
+      workload = job_workload(job)
+
+      batches = chain_batches(rules, job_chains, workload)
+      batches = add_batch_deps(batches)
+      batches = add_rules_and_consolidate(rules, batches)
+    end
+  end
+end

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -0,0 +1,173 @@
+module HPC
+  module Orchestration
+    def self.check_chains(chains, job)
+      matches = []
+      chains.each do |name, chain|
+        next unless chain[:tasks].include?(job.workflow.to_s)
+        next unless chain[:tasks][job.workflow.to_s].include?(job.task_name.to_s)
+        matches << name
+      end
+      matches
+    end
+
+    def self.parse_chains(rules)
+      return {} if rules["chains"].nil?
+
+      chains = IndiferentHash.setup({})
+      rules["chains"].each do |name,rules|
+        rules  = IndiferentHash.setup(rules.dup)
+        chain_tasks = rules.delete(:tasks).split(/,\s*/)
+        workflow = rules.delete(:workflow)
+
+        chain_tasks.each do |task|
+          chain_workflow, chain_task = task.split("#")
+          chain_task, chain_workflow = chain_workflow, workflow if chain_task.nil? or chain_tasks.empty?
+
+          chains[name] ||= IndiferentHash.setup({:tasks => {}, :rules => rules })
+          chains[name][:tasks][chain_workflow] ||= []
+          chains[name][:tasks][chain_workflow] << chain_task
+        end
+      end
+
+      chains
+    end
+
+    def self.job_dependencies(job)
+      (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
+    end
+
+    #def self.job_workload(job)
+    #  workload = []
+    #  heap = []
+    #  heap << job
+    #  while job = heap.pop
+    #    next if job.done?
+    #    workload << job
+    #    heap.concat job_dependencies(job)
+    #    heap.uniq!
+    #  end
+    #  workload.uniq
+    #end
+
+    #def self.top_level_job(jobs)
+    #  top = jobs.select do |job|
+    #    (jobs - job_workload(job)).empty? &&
+    #      (job_workload(job) - jobs).select{|j| (job_workload(j) & jobs).any? }.empty?
+    #  end
+    #  return nil if top.length != 1
+    #  top.first
+    #end
+
+    #def self.job_chains(rules, job)
+    #  workload = job_workload(job)
+    #  chains = parse_chains(rules)
+
+    #  chain_jobs = {}
+    #  workload.each do |job|
+    #    check_chains(chains, job).each do |match|
+    #      chain_jobs[match] ||= []
+    #      chain_jobs[match] << job
+    #    end
+    #  end
+
+    #  job_chains = []
+
+    #  seen = []
+    #  chain_jobs.sort_by{|name,jobs| jobs.length }.reverse.each do |name,jobs|
+    #    remain = jobs - seen
+    #    next unless remain.length > 1
+    #    top_level_job = top_level_job(jobs)
+    #    next if top_level_job.nil?
+    #    job_chains << {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
+    #    seen.concat remain
+    #  end
+
+    #  job_chains
+    #end
+
+    #def self._job_chains(rules, job)
+    #  workload = job_workload(job)
+    #  chains = parse_chains(rules)
+
+    #  matches = check_chains(chains, job)
+
+    #  job_chains = {}
+    #  job.dependencies.each do |dep|
+    #    dep_chains = _job_chains(rules, dep)
+    #    matches.each do |match|
+    #      if dep_chains[match] && dep_chains[match].include?(dep)
+    #        dep_chains[match].prepend job
+    #      end
+    #    end
+    #    job_chains.merge!(dep_chains)
+    #  end
+
+    #  matches.each do |match|
+    #    job_chains[match] ||= [job]
+    #  end
+
+    #  job_chains
+    #end
+
+    #def self.job_chains(rules, job)
+    #  job_chains = self._job_chains(rules, job)
+    #  iif job_chains
+    #  chains = parse_chains(rules)
+
+    #  seen = []
+    #  job_chains.collect do |name,jobs|
+    #    remain = jobs - seen
+    #    next unless remain.length > 1
+    #    top_level_job = top_level_job(jobs)
+    #    next if top_level_job.nil?
+    #    seen.concat remain
+    #    {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
+    #  end.compact
+    #end
+
+    def self.job_chains(rules, job)
+      chains = self.parse_chains(rules)
+
+      matches = check_chains(chains, job)
+
+      dependencies = job_dependencies(job)
+
+      job_chains = []
+      new_job_chains = {}
+      dependencies.each do |dep|
+        dep_matches = check_chains(chains, dep)
+        common = matches & dep_matches
+
+        dep_chains = job_chains(rules, dep)
+        found = []
+        dep_chains.each do |match,info|
+          if common.include?(match)
+            found << match
+            new_info = new_job_chains[match] ||= {}
+            new_info[:jobs] ||= []
+            new_info[:jobs].concat info[:jobs]
+            new_info[:top_level] = job
+          else
+            job_chains << [match, info]
+          end
+        end
+
+        (common - found).each do |match|
+          info = {}
+          info[:jobs] = [job, dep]
+          info[:top_level] = job
+          job_chains << [match, info]
+        end
+      end
+
+      new_job_chains.each do |match,info|
+        info[:jobs].prepend job
+        job_chains << [match, info]
+      end
+
+      job_chains
+    end
+
+  end
+end
+

data/lib/rbbt/hpc/orchestrate/rules.rb

@@ -0,0 +1,70 @@
+module HPC
+  module Orchestration
+    def self.add_config_keys(current, new)
+      if current.nil?
+        new
+      else
+        new + ',' + current
+      end.gsub(/,\s*/,',').split(",").reverse.uniq.reverse * ","
+    end
+
+    def self.workflow_rules(rules, workflow)
+      return {} if rules[workflow].nil?
+      return {} if rules[workflow]["defaults"].nil? 
+      IndiferentHash.setup(rules[workflow]["defaults"])
+    end
+
+    def self.merge_rules(current, new)
+      return IndiferentHash.setup({}) if (new.nil? || new.empty?) && (current.nil? || current.empty?)
+      return IndiferentHash.setup(current.dup) if new.nil? || new.empty?
+      return IndiferentHash.setup(new.dup) if current.nil? || current.empty?
+      target = IndiferentHash.setup(current.dup)
+      new.each do |k,value|
+        case k.to_s
+        when "config_keys"
+          target[k] = add_config_keys target["config_keys"], value
+        else
+          next if target.include?(k)
+          target[k] = value
+        end
+      end
+      target
+    end
+
+    def self.accumulate_rules(current, new)
+      return IndiferentHash.setup({}) if (new.nil? || new.empty?) && (current.nil? || current.empty?)
+      return IndiferentHash.setup(current.dup) if new.nil? || new.empty?
+      return IndiferentHash.setup(new.dup) if current.nil? || current.empty?
+      target = IndiferentHash.setup(current.dup)
+      new.each do |k,value|
+        case k.to_s
+        when "config_keys"
+          target[k] = add_config_keys target["config_keys"], value
+        when "cpus"
+          target[k] = [target[k], value].compact.sort_by{|v| v.to_i}.last
+        when "time"
+          target[k] = Misc.format_seconds [target[k], value].compact.inject(0){|acc,t|  acc += Misc.timespan t }
+        when "skip"
+          skip = target[k] && value
+          target.delete k unless skip
+        else
+          next if target.include?(k)
+          target[k] = value
+        end
+      end
+      target
+    end
+
+    def self.task_specific_rules(rules, workflow, task)
+      defaults = rules[:defaults] || {}
+      workflow = workflow.to_s
+      task = task.to_s
+      return defaults if rules[workflow].nil?
+      workflow_rules = merge_rules(workflow_rules(rules, workflow), defaults)
+      return IndiferentHash.setup(workflow_rules.dup) if rules[workflow][task].nil?
+      merge_rules(rules[workflow][task], workflow_rules)
+    end
+
+
+  end
+end