rbbt-util 5.28.9 → 5.29.0

data/lib/rbbt/workflow/examples.rb

@@ -50,8 +50,8 @@ module Workflow
         case input_types[input]
         when :file
           Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.read$/
-            inputs[input.to_sym]  = Open.read(file)
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             inputs[input.to_sym]  = Open.realpath(file)
           end

data/lib/rbbt/workflow/integration/ansible.rb

@@ -0,0 +1,53 @@
+require_relative 'ansible/workflow'
+require 'rbbt/workflow/usage'
+
+module Ansible
+  def self.play(playbook, inventory = nil)
+    inventory = Rbbt.etc.ansible_inventory.find
+    Log.with_severity 0 do
+      TmpFile.with_file do |tmp|
+        if Hash === playbook
+          Open.write(tmp, [playbook].to_yaml)
+          playbook = tmp
+        end
+        CMD.cmd_log("ansible-playbook -i #{inventory} #{playbook}")
+      end
+    end
+  end
+
+  def self.clean_symbols(hash)
+    new = {}
+    hash.each do |key,value|
+      key = key.to_s
+      value = case value
+              when Symbol
+                value.to_s
+              when Hash
+                self.clean_symbols(value)
+              else
+                value
+              end
+      new[key] = value
+    end
+    new
+  end
+
+  def self.workflow2playbook(workflow, task, options = {})
+    job_options = workflow.get_SOPT(workflow.tasks[task])
+
+    tasks = workflow.job(task, nil, job_options).exec
+
+    hosts = options[:hosts] || 'localhost'
+
+    clean_tasks = tasks.collect{|task| self.clean_symbols task }
+    {"hosts" => hosts, "tasks" => clean_tasks}
+  end
+
+  def self.playbook(file, task = nil, options = {})
+    task = 'default' if task.nil?
+
+    workflow = Workflow.require_workflow file
+    task = workflow.tasks.keys.last if workflow.tasks[task].nil?
+    workflow2playbook workflow, task, options
+  end
+end

data/lib/rbbt/workflow/integration/ansible/workflow.rb

@@ -0,0 +1,60 @@
+require 'rbbt/workflow'
+
+module Ansible
+  module AnsibleWorkflow
+    def self.extended(object)
+      class << object
+        attr_accessor :ans_tasks
+      end
+
+      object.helper :register do |task_info|
+        desc = task.description if task
+        name ||= desc || short_path
+        task_info = {"name" => name}.merge(task_info)
+        @ans_tasks ||= []
+        @ans_tasks << task_info
+        task
+      end
+
+      object.helper :ans do |name, info|
+        register({ name => info})
+      end
+
+      object.helper :add do |name, info|
+        @ans_tasks.last[name.to_s] = info
+      end
+
+      object.helper :shell do |cmd|
+        register({"shell" => cmd.strip})
+      end
+
+      object.helper :sudo do |cmd|
+        register({"shell" => cmd.strip, "become" => 'yes'})
+      end
+
+      object.helper :singularity do |scmd|
+        img = config :singularity_img, :build, :test, :small, :default => '/data/img/singularity/rbbt/rbbt.simg'
+        container = config :singularity_container, :build, :test, :small, :default => '/data/img/sandbox/mvazque2/'
+        cmd = <<-EOF
+singularity exec -C -H '#{container}' '#{img}' #{scmd}
+        EOF
+        register({"shell" => cmd.strip, "name" => short_path})
+      end
+
+
+      object.helper :produce_task do
+        @ans_tasks
+      end
+    end
+
+    def play(name = nil, &block)
+      name = Misc.snake_case(@description) if name.nil?
+      task name => :yaml do |*args|
+        self.instance_exec *args, &block
+        dependencies.inject([]){|acc,dep| acc += dep.load } + produce_task
+      end
+    end
+
+  end
+end
+

data/lib/rbbt/workflow/step.rb

@@ -12,6 +12,9 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name
+
+  RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
   class << self
     attr_accessor :lock_dir
@@ -143,17 +146,24 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      dep_info[:inputs].each do |k,v|
-        all_inputs[k] = v unless all_inputs.include?(k)
-      end if dep_info[:inputs]
-      deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-      deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
       seen << path
     end
 
     all_inputs
   end
 
+  def dependencies=(dependencies)
+    @dependencies = dependencies
+    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
+  end
+
   def recursive_inputs
     if NamedArray === inputs
       i = {}
@@ -449,6 +459,7 @@ class Step
     status << "not running" if ! done? && ! running? 
     status.unshift " " if status.any?
     Log.high "Cleaning step: #{path}#{status * " "}"
+    Log.stack caller if RBBT_DEBUG_CLEAN
     abort if ! done? && running?
     Step.clean(path)
     self

data/lib/rbbt/workflow/step/accessor.rb

@@ -8,6 +8,16 @@ class Step
                        end
                      end
 
+  def self.serialize_info(info)
+    info = info.clean_version if IndiferentHash === info
+    INFO_SERIALIZER.dump(info)
+  end
+
+  def self.load_serialized_info(io)
+    IndiferentHash.setup(INFO_SERIALIZER.load(io))
+  end
+
+
   def self.wait_for_jobs(jobs)
     jobs = [jobs] if Step === jobs
     begin
@@ -59,7 +69,7 @@ class Step
   def self.step_info(path)
     begin
       Open.open(info_file(path), :mode => 'rb') do |f|
-        INFO_SERIALIZER.load(f)
+        self.load_serialized_info(f)
       end
     rescue Exception
       Log.exception $!
@@ -83,18 +93,18 @@ class Step
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
-      when Array === value
-        Open.write(path, value * "\n")
-      when IO === value
-        Open.write(path, value)
-      when type == "file"
+      when Step === value
+        Open.link(value.path, path)
+      when type.to_s == "file"
         if String === value && File.exists?(value)
           Open.link(value, path)
         else
-          Open.write(path + '.read', value.to_s)
+          Open.write(path + '.yaml', value.to_yaml)
         end
-      when Step === value
-        value = value.produce.load 
+      when Array === value
+        Open.write(path, value * "\n")
+      when IO === value
+        Open.write(path, value)
       else
         Open.write(path, value.to_s)
       end
@@ -104,7 +114,7 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = job.task_name
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     task_info = workflow.task_info(task_name)
@@ -113,15 +123,17 @@ class Step
     input_defaults = task_info[:input_defaults]
 
     inputs = {}
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
-      next if options and ! options.include?(name)
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
       next if value.nil?
       next if input_defaults[name] == value
       inputs[name] = value
     end
 
-    if options.include? 'override_dependencies'
+    if options && options.include?('override_dependencies')
       inputs.merge!(:override_dependencies => open[:override_dependencies])
       input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
     end
@@ -188,7 +200,7 @@ class Step
                 info_lock.lock if check_lock and false
                 begin
                   Open.open(info_file, :mode => 'rb') do |file|
-                    INFO_SERIALIZER.load(file) #|| {}
+                    Step.load_serialized_info(file)
                   end
                 ensure
                   info_lock.unlock if check_lock and false
@@ -204,7 +216,7 @@ class Step
       Log.debug{"Error loading info file: " + info_file}
       Log.exception $!
       Open.rm info_file
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump({:status => :error, :messages => ["Info file lost"]}))
+      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
       raise $!
     end
   end
@@ -212,10 +224,10 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
-      @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump(i), :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
+      @info_cache = IndiferentHash.setup(i)
       @info_cache_time = Time.now
     end
   end
@@ -227,9 +239,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
       i[key] = value 
-      @info_cache = i
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      dump = Step.serialize_info(i)
+      @info_cache = IndiferentHash.setup(i)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end
@@ -242,9 +254,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false)
       i.merge! hash
-      @info_cache = i
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      dump = Step.serialize_info(i)
+      @info_cache = IndiferentHash.setup(i)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end
@@ -537,7 +549,7 @@ class Step
   end
 
   def file(name)
-    Path.setup(File.join(files_dir, name.to_s))
+    Path.setup(File.join(files_dir, name.to_s), workflow, self)
   end
 
   def save_file(name, content)

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,13 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
+        if ! (job.resumable? && (job.updated? && ! job.dirty?))
+          Log.high "About to clean -- status: #{status}, present #{File.exists?(job.path)}, " +
+            %w(done? error? recoverable_error? noinfo? updated? dirty? aborted? running? resumable?).
+            collect{|v| [v, job.send(v)]*": "} * ", " if RBBT_DEBUG_CLEAN
+
+          job.clean
+        end
         job.set_info :status, :cleaned
       end
 
@@ -121,7 +127,7 @@ class Step
   end
 
   def input_dependencies
-    inputs.flatten.select{|i| Step === i}
+    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
 

data/lib/rbbt/workflow/step/run.rb

@@ -112,7 +112,7 @@ class Step
   end
 
   def updatable?
-    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?))
+    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?)) || (ENV["RBBT_UPDATE"] && ! (done? && ! Open.exists?(info_file)))
   end
 
   def dependency_checks
@@ -128,7 +128,7 @@ class Step
   end
 
   def input_checks
-    inputs.select{|i| Step === i }.
+    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource}).
       select{|dependency| dependency.updatable? }
   end
 
@@ -154,25 +154,28 @@ class Step
     canfail_paths = self.canfail_paths
     this_mtime = Open.mtime(self.path) if Open.exists?(self.path)
 
-    checks.each do |dep| 
-      next unless dep.updatable?
-      dep_done = dep.done?
+    outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    outdated_dep = checks.reject{|dep| dep.done? || (dep.error? && ! dep.recoverable_error? && canfail_paths.include?(dep.path)) }
 
-      begin
-        if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
-          outdated_time << dep
-        end
-      rescue
-      end
+    #checks.each do |dep| 
+    #  next unless dep.updatable?
+    #  dep_done = dep.done?
 
-      # Is this pointless? this would mean some dep got updated after a later
-      # dep but but before this one.
-      #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+    #  begin
+    #    if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
+    #      outdated_time << dep
+    #    end
+    #  rescue
+    #  end
 
-      if (! dep_done && ! canfail_paths.include?(dep.path))
-        outdated_dep << dep
-      end
-    end
+    #  # Is this pointless? this would mean some dep got updated after a later
+    #  # dep but but before this one.
+    #  #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+
+    #  if (! dep_done && ! canfail_paths.include?(dep.path))
+    #    outdated_dep << dep
+    #  end
+    #end
 
     Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
     Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
@@ -215,7 +218,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type
+        result_type = @task.result_type if @task
         result_type = info[:result_type] if result_type.nil?
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step

data/lib/rbbt/workflow/util/archive.rb

@@ -67,8 +67,10 @@ class Step
       next unless File.exists?(step.path)
       job_files << step.path
       job_files << step.info_file if File.exists?(step.info_file)
+      job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
       job_file_dir_content = Dir.glob(step.files_dir + '/**/*')
       job_files += job_file_dir_content
+      job_files << step.files_dir if File.exists?(step.files_dir)
       rec_dependencies = Set.new
 
       next unless recursive

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -26,15 +26,25 @@ module Workflow
       workload
     end
 
+    def self.workload(jobs)
+      jobs.inject({}) do |acc,job| 
+          Orchestrator.job_workload(job).each do |j,d|
+            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+          end
+          acc
+        end
+    end
+
     def self.job_rules(rules, job)
       workflow = job.workflow.to_s
       task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
 
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+      return IndiferentHash.setup(defaults) unless rules[workflow]
+      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
 
       job_rules = IndiferentHash.setup(rules[workflow][task_name])
-      rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
+      defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
 
@@ -143,18 +153,25 @@ module Workflow
       end
     end
 
-    def erase_job_dependencies(job, rules, workload, top_level_jobs)
+    def erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
       job.dependencies.each do |dep|
         next if top_level_jobs.include? dep.path
         next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true'
 
-        list = (workload.keys - [job]).collect{|pending| pending.dependencies}.flatten
-        next if list.include?(dep)
+        dep_path = dep.path
+        parents = all_jobs.select do |parent| 
+          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last }
+          paths.include? dep_path
+        end
+
+        next unless parents.reject{|parent| parent.done? }.empty?
 
-        Log.high "Erasing #{dep.path} from #{job.path}"
-        job.archive_deps
-        job.copy_files_dir
-        job.dependencies = job.dependencies - [dep]
+        parents.each do |parent|
+          Log.high "Erasing #{dep.path} from #{parent.path}"
+          parent.archive_deps
+          parent.copy_files_dir
+          parent.dependencies = parent.dependencies - [dep]
+        end
         dep.clean
       end
     end
@@ -162,7 +179,8 @@ module Workflow
     def process(rules, jobs)
       begin
 
-        workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
+        workload = Orchestrator.workload(jobs)
+        all_jobs = workload.keys
 
         top_level_jobs = jobs.collect{|job| job.path }
         while workload.any? 
@@ -187,7 +205,7 @@ module Workflow
             when job.done?
               Log.debug "Orchestrator done #{job.path}"
               release_resources(job)
-              erase_job_dependencies(job, rules, workload, top_level_jobs)
+              erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
 
             when job.running?
               next