rbbt-util 5.32.24 → 5.32.28

data/lib/rbbt/hpc/orchestrate.old.rb

@@ -0,0 +1,220 @@
+require 'rbbt/workflow/util/orchestrator'
+module HPC
+  module Orchestration
+    def job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+      task_name = job.overriden.to_s if Symbol === job.overriden
+
+      defaults = rules["defaults"] || {}
+      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
+
+      job_rules = IndiferentHash.setup(defaults.dup)
+
+      rules["chains"].each do |name,info|
+        IndiferentHash.setup(info)
+        chain_tasks = info[:tasks].split(/,\s*/)
+
+        chain_tasks.each do |task|
+          task_workflow, chain_task = task.split("#")
+          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
+          job_rules["chain_tasks"] ||= {}
+          job_rules["chain_tasks"][task_workflow] ||= []
+          job_rules["chain_tasks"][task_workflow]  << chain_task
+          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
+          config_keys = job_rules.delete :config_keys
+          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
+          if config_keys
+            config_keys.gsub!(/,\s+/,',') 
+            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+          end
+        end
+
+        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
+          break
+        else
+          job_rules.delete "chain_tasks" 
+        end
+      end if rules["chains"]
+
+      config_keys = job_rules.delete :config_keys
+      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
+
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
+      end
+
+      if rules["skip"] && rules["skip"][workflow]
+        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
+      end
+
+      job_rules
+    end
+
+    def get_job_dependencies(job, job_rules = nil)
+      deps = job.dependencies || []
+      deps += job.input_dependencies || []
+      deps
+    end
+
+    def get_recursive_job_dependencies(job)
+      deps = get_job_dependencies(job) 
+      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
+    end
+
+    def piggyback(job, job_rules, job_deps)
+      return false unless job_rules["skip"]
+      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
+      final_deps = final_deps.reject{|dep| dep.done? }
+      return final_deps.first if final_deps.length == 1
+      return false
+    end
+
+    def get_chains(job, rules, chains = {})
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      input_deps = []
+      job.rec_dependencies.each do |dep|
+        input_deps.concat dep.input_dependencies
+      end
+
+      job_deps.each do |dep|
+        input_deps.concat dep.input_dependencies
+        get_chains(dep, rules, chains)
+      end
+
+      job_deps.select do |dep|
+        chained = job_rules["chain_tasks"] &&
+          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
+          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
+
+        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
+        chained || dep_skip
+      end.each do |dep|
+        chains[job] ||= [] 
+        chains[job] << dep 
+        chains[job].concat chains[dep] if chains[dep]
+        chains[job].uniq!
+      end
+
+      chains
+    end
+
+    def workload(job, rules, chains, options, seen = nil)
+      return [] if job.done?
+      if seen.nil?
+        seen = {}
+        target_job = true
+      end
+
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+      chain = chains[job]
+      chain = chain.reject{|j| seen.include? j.path} if chain
+      chain = chain.reject{|dep| dep.done? } if chain
+      piggyback = piggyback(job, job_rules, job_deps)
+      dep_ids = job_deps.collect do |dep|
+        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
+        next_options = IndiferentHash.setup(options.dup)
+        if piggyback and piggyback == dep
+          next_options[:piggyback] ||= []
+          next_options[:piggyback].push job
+          ids = workload(dep, rules, chains, next_options, seen)
+        else
+          next_options.delete :piggyback
+          ids = workload(dep, rules, chains, next_options, seen)
+        end
+
+        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
+
+        seen[dep.path] = ids
+        ids
+      end.compact.flatten.uniq
+
+      return seen[job.path] || dep_ids if seen.include?(job.path)
+
+      if piggyback and seen[piggyback.path]
+        return seen[job.path] = seen[piggyback.path] 
+      end
+
+      job_rules.delete :chain_tasks
+      job_rules.delete :tasks
+      job_rules.delete :workflow
+      
+
+      option_config_keys = options[:config_keys]
+
+      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
+      job_options.delete :orchestration_rules
+
+      config_keys = job_rules.delete(:config_keys)
+      if config_keys
+        config_keys.gsub!(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
+      end
+
+      if option_config_keys
+        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
+        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
+      end
+
+      if options[:piggyback]
+        manifest = options[:piggyback].uniq
+        manifest += [job]
+        manifest.concat chain if chain
+
+        job = options[:piggyback].first
+
+        job_rules = self.job_rules(rules, job)
+        new_config_keys = self.job_rules(rules, job)[:config_keys]
+        if new_config_keys
+          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
+          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
+        end
+
+        job_options.delete :piggyback
+      else
+        manifest = [job]
+        manifest.concat chain if chain
+      end
+
+      manifest.uniq!
+
+      job_options[:manifest] = manifest.collect{|j| j.task_signature }
+
+      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
+
+      if options[:dry_run]
+        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+        job_options[:manifest].first
+      else
+        run_job(job, job_options)
+      end
+    end
+
+
+    def orchestrate_job(job, options)
+      options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
+      options.delete "tail"
+      options.delete "printpath"
+      options.delete "detach"
+      options.delete "jobname"
+
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules ||= {}
+      IndiferentHash.setup(rules)
+
+      chains = get_chains(job, rules)
+
+      workload(job, rules, chains, options)
+    end
+
+  end
+end
+

data/lib/rbbt/hpc/orchestrate.rb

@@ -1,203 +1,10 @@
-require 'rbbt/workflow/util/orchestrator'
+#require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/hpc/orchestrate/rules'
+require 'rbbt/hpc/orchestrate/chains'
+require 'rbbt/hpc/orchestrate/batches'
 module HPC
   module Orchestration
 
-    def job_rules(rules, job)
-      workflow = job.workflow.to_s
-      task_name = job.task_name.to_s
-      task_name = job.overriden.to_s if Symbol === job.overriden
-
-      defaults = rules["defaults"] || {}
-      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
-
-      job_rules = IndiferentHash.setup(defaults.dup)
-
-      rules["chains"].each do |name,info|
-        IndiferentHash.setup(info)
-        chain_tasks = info[:tasks].split(/,\s*/)
-
-        chain_tasks.each do |task|
-          task_workflow, chain_task = task.split("#")
-          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
-          job_rules["chain_tasks"] ||= {}
-          job_rules["chain_tasks"][task_workflow] ||= []
-          job_rules["chain_tasks"][task_workflow]  << chain_task
-          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
-          config_keys = job_rules.delete :config_keys
-          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
-          if config_keys
-            config_keys.gsub!(/,\s+/,',') 
-            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-          end
-        end
-
-        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
-          break
-        else
-          job_rules.delete "chain_tasks" 
-        end
-      end if rules["chains"]
-
-      config_keys = job_rules.delete :config_keys
-      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
-
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-      end
-
-      if rules["skip"] && rules["skip"][workflow]
-        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
-      end
-
-      job_rules
-    end
-
-    def get_job_dependencies(job, job_rules = nil)
-      deps = job.dependencies || []
-      deps += job.input_dependencies || []
-      deps
-    end
-
-    def get_recursive_job_dependencies(job)
-      deps = get_job_dependencies(job) 
-      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
-    end
-
-    def piggyback(job, job_rules, job_deps)
-      return false unless job_rules["skip"]
-      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
-      final_deps = final_deps.reject{|dep| dep.done? }
-      return final_deps.first if final_deps.length == 1
-      return false
-    end
-
-    def get_chains(job, rules, chains = {})
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      input_deps = []
-      job.rec_dependencies.each do |dep|
-        input_deps.concat dep.input_dependencies
-      end
-
-      job_deps.each do |dep|
-        input_deps.concat dep.input_dependencies
-        get_chains(dep, rules, chains)
-      end
-
-      job_deps.select do |dep|
-        chained = job_rules["chain_tasks"] &&
-          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
-          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
-
-        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
-        chained || dep_skip
-      end.each do |dep|
-        chains[job] ||= [] 
-        chains[job] << dep 
-        chains[job].concat chains[dep] if chains[dep]
-        chains[job].uniq!
-      end
-
-      chains
-    end
-
-    def workload(job, rules, chains, options, seen = nil)
-      return [] if job.done?
-      if seen.nil?
-        seen = {}
-        target_job = true
-      end
-
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      chain = chains[job]
-      chain = chain.reject{|j| seen.include? j.path} if chain
-      chain = chain.reject{|dep| dep.done? } if chain
-      piggyback = piggyback(job, job_rules, job_deps)
-      dep_ids = job_deps.collect do |dep|
-        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
-        next_options = IndiferentHash.setup(options.dup)
-        if piggyback and piggyback == dep
-          next_options[:piggyback] ||= []
-          next_options[:piggyback].push job
-          ids = workload(dep, rules, chains, next_options, seen)
-        else
-          next_options.delete :piggyback
-          ids = workload(dep, rules, chains, next_options, seen)
-        end
-
-        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
-
-        seen[dep.path] = ids
-        ids
-      end.compact.flatten.uniq
-
-      return seen[job.path] || dep_ids if seen.include?(job.path)
-
-      if piggyback and seen[piggyback.path]
-        return seen[job.path] = seen[piggyback.path] 
-      end
-
-      job_rules.delete :chain_tasks
-      job_rules.delete :tasks
-      job_rules.delete :workflow
-      
-
-      option_config_keys = options[:config_keys]
-
-      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
-      job_options.delete :orchestration_rules
-
-      config_keys = job_rules.delete(:config_keys)
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
-      end
-
-      if option_config_keys
-        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
-      end
-
-      if options[:piggyback]
-        manifest = options[:piggyback].uniq
-        manifest += [job]
-        manifest.concat chain if chain
-
-        job = options[:piggyback].first
-
-        job_rules = self.job_rules(rules, job)
-        new_config_keys = self.job_rules(rules, job)[:config_keys]
-        if new_config_keys
-          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
-          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
-        end
-
-        job_options.delete :piggyback
-      else
-        manifest = [job]
-        manifest.concat chain if chain
-      end
-
-      manifest.uniq!
-
-      job_options[:manifest] = manifest.collect{|j| j.task_signature }
-
-      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
-
-      if options[:dry_run]
-        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
-        job_options[:manifest].first
-      else
-        run_job(job, job_options)
-      end
-    end
-
-
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -211,9 +18,26 @@ module HPC
       rules ||= {}
       IndiferentHash.setup(rules)
 
-      chains = get_chains(job, rules)
-
-      workload(job, rules, chains, options)
+      batches = HPC::Orchestration.job_batches(rules, job)
+
+      batch_ids = {}
+      while batches.any?
+        top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
+        raise "No batch without unmet dependencies" if top.nil?
+        batches.delete top
+        job_options = options.merge(top[:rules])
+        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+        job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
+
+        if options[:dry_run]
+          puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          batch_ids[top] = top[:top_level].task_signature
+        else
+          id = run_job(top[:top_level], job_options)
+          batch_ids[top] = id
+        end
+      end
     end
 
   end

data/lib/rbbt/hpc/slurm.rb

@@ -51,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "nodes" => nodes,
                        "time" => time,
                        "exclusive" => exclusive,
+                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,

data/lib/rbbt/persist/tsv.rb

@@ -78,7 +78,7 @@ module Persist
     end
   end
 
-  def self.persist_tsv(source, filename, options = {}, persist_options = {}, &block)
+  def self.persist_tsv(source, filename = nil, options = {}, persist_options = {}, &block)
     persist_options[:prefix] ||= "TSV"
 
     if data = persist_options[:data]

data/lib/rbbt/tsv/excel.rb

@@ -177,7 +177,7 @@ module TSV
 
       sheet ||= "0"
       workbook = RubyXL::Parser.parse file
-      if sheet &&  sheet =~ /^\d+$/
+      if sheet && sheet =~ /^\d+$/
         sheet = workbook.worksheets.collect{|s| s.sheet_name }[sheet.to_i]
       end
       sheet_name = sheet
@@ -185,7 +185,9 @@ module TSV
 
       TmpFile.with_file :extension => Misc.sanitize_filename(sheet_name.to_s) do |filename|
 
-        sheet    = sheet ? workbook[sheet] : workbook.worksheets.first
+        sheet    = sheet_name ? workbook[sheet_name] : workbook.worksheets.first
+
+        raise "No sheet #{sheet_name} found" if sheet.nil?
 
         rows = []
 
@@ -217,21 +219,27 @@ module TSV
     end
 
     def self.write(tsv, file, options = {})
-      sheet = Misc.process_options options, :sheet
+      sheet, add_sheet = Misc.process_options options, :sheet, :add_sheet
 
       fields, rows = TSV._excel_data(tsv, options)
 
-      book = RubyXL::Workbook.new
-      sheet1 = book.worksheets.first
-      sheet1.sheet_name = sheet if sheet
+      if Open.exists?(file) && add_sheet
+        book = RubyXL::Parser.parse file
+        sheet1 = book.add_worksheet(sheet)
+      else
+        book = RubyXL::Workbook.new
+        sheet1 = book.worksheets.first
+        sheet1.sheet_name = sheet if sheet
+      end
 
       fields.each_with_index do |e,i|
         sheet1.add_cell(0, i, e)
-      end
+      end if fields
 
       rows.each_with_index do |cells,i|
+        i += 1 if fields
         cells.each_with_index do |e,j|
-          sheet1.add_cell(i+1, j, e)
+          sheet1.add_cell(i, j, e)
         end
       end
 

data/lib/rbbt/util/log.rb

@@ -108,8 +108,9 @@ module Log
 
 
   def self._ignore_stderr
-    backup_stderr = STDERR.dup
-    File.open('/dev/null', 'w') do |f|
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
         STDERR.reopen(f)
         begin
           yield
@@ -117,6 +118,9 @@ module Log
           STDERR.reopen backup_stderr
           backup_stderr.close
         end
+      end
+    rescue Errno::ENOENT
+      yield
     end
   end
 

data/lib/rbbt/util/migrate.rb

@@ -49,6 +49,7 @@ puts resource[path].find(search_path)
     test_str = options[:test] ? '-nv' : ''
 
     real_paths.each do |source_path|
+      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}"
       if File.directory?(source_path) || source_path =~ /\/$/
         source_path += "/" unless source_path[-1] == "/"
         target += "/" unless target[-1] == "/"
@@ -76,8 +77,12 @@ puts resource[path].find(search_path)
           files_from_str = ""
         end
 
-        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+        #cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+         
+        # rsync_args = "-avztAXHP --copy-unsafe-links"
+        rsync_args = "-avztAHP --copy-unsafe-links"
 
+        cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
 
         cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
 

data/lib/rbbt/util/misc/inspect.rb

@@ -280,8 +280,11 @@ module Misc
       i = parts.index job
       begin
         workflow, task = parts.values_at i - 2, i - 1
-        return Kernel.const_get(workflow).tasks.include? task.to_sym
+        Workflow.require_workflow workflow
+        #return Kernel.const_get(workflow).tasks.include? task.to_sym
+        return true
       rescue
+        Log.exception $!
       end
     end
     false

data/lib/rbbt/util/misc.rb

@@ -70,6 +70,11 @@ module Misc
   end
 
   def self.timespan(str, default = "s")
+    if str.include?(":")
+      seconds, minutes, hours = str.split(":").reverse
+      return seconds.to_i + minutes.to_i * 60 + hours.to_i * 60 * 60
+    end
+
     tokens = {
       "s" => (1),
       "sec" => (1),

data/lib/rbbt/util/python.rb

@@ -52,7 +52,7 @@ module RbbtPython
       if Array === imports
         pyfrom mod, :import => imports
       elsif Hash === imports
-        pyimport mod, imports
+        pyimport mod, **imports
       else
         pyimport mod 
       end

data/lib/rbbt/workflow/definition.rb

@@ -76,7 +76,7 @@ module Workflow
     extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
-      raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join