scout-gear 10.9.0 → 10.10.1

data/lib/scout/workflow/deployment/scheduler/lfs.rb

@@ -0,0 +1,125 @@
+require_relative 'job'
+require 'scout'
+
+module LSF 
+  extend SchedulerJob
+
+  def self.system
+    "LSF"
+  end
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $LSB_MAX_NUM_PROCESSORS )"
+[ ! -z $LSB_MAX_MEM_RUSAGE ] && let MAX_MEMORY="$LSB_MAX_MEM_RUSAGE" || MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$LSF_JOBID
+export BATCH_SYSTEM=#{system}
+    EOF
+  end
+
+  def self.header(options = {})
+    options        = options.dup
+
+    queue          = IndiferentHash.process_options options, :queue
+    task_cpus      = IndiferentHash.process_options options, :task_cpus
+    time           = IndiferentHash.process_options options, :time
+    nodes          = IndiferentHash.process_options options, :nodes
+    workdir        = IndiferentHash.process_options options, :workdir
+    exclusive      = IndiferentHash.process_options options, :exclusive
+
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+    batch_name ||= File.basename(batch_dir)
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    time = time.split(":").values_at(0, 1) * ":"
+
+    header =<<-EOF
+#!/bin/bash
+#BSUB -J "#{batch_name}"
+#BSUB -cwd "#{workdir}"
+#BSUB -oo "#{fout}"
+#BSUB -eo "#{ferr}"
+#BSUB -q "#{queue}"
+#BSUB -n "#{task_cpus}"
+#BSUB -W "#{time}"
+    EOF
+
+    header << "#BSUB -x" << "\n" if exclusive
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    STDERR.puts Log.color(:magenta, "Issuing LSF file: #{fcmd}")
+    STDERR.puts Open.read(fcmd)
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_list = dependencies && dependencies.any? ? dependencies.collect{|d| "post_done(#{d})"} : []
+      canfail_dep_list = canfail_dependencies && canfail_dependencies.any? ? canfail_dependencies.collect{|d| "done(#{d})"} : []
+
+      dep_list = normal_dep_list + canfail_dep_list
+
+      if dep_list.any?
+        dep_str = '-w "' + dep_list * " && " + '"'
+      else
+        dep_str = ""
+      end
+
+      cmd = "bsub #{dep_str} < '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, cmd)
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local scout): ") + Log.color(:blue, "scout lsf tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "BSUB job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("bjobs -w").read
+    else
+      CMD.cmd("bjobs -w #{job}").read
+    end
+  end
+end
+

data/lib/scout/workflow/deployment/scheduler/pbs.rb

@@ -0,0 +1,176 @@
+require_relative 'job'
+require 'scout'
+
+module PBS 
+  extend SchedulerJob
+
+  def self.system
+    "PBS"
+  end
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $PBS_CPUS_PER_TASK )"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK" 
+[ ! -z $PBS_MEM_PER_NODE ] && MAX_MEMORY="$PBS_MEM_PER_NODE"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$PBS_JOBID
+export BATCH_SYSTEM=#{system}
+
+cd ${PBS_O_WORKDIR}
+    EOF
+  end
+
+  def self.header(options = {})
+    options = options.dup
+
+    workdir    = IndiferentHash.process_options options, :workdir
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+
+    queue      = IndiferentHash.process_options options, :queue
+    account    = IndiferentHash.process_options options, :account
+    time       = IndiferentHash.process_options options, :time
+    nodes      = IndiferentHash.process_options options, :nodes
+
+    # PBS 
+    place      = IndiferentHash.process_options options, :place, :place => 'scatter'
+    system     = IndiferentHash.process_options options, :partition
+    filesystems = IndiferentHash.process_options options, :filesystems
+
+    filesystems = "home" if filesystems.nil?
+
+    filesystems = filesystems * "," if Array === filesystems
+
+    # NOT USED 
+    partition  = IndiferentHash.process_options options, :partition
+    task_cpus  = IndiferentHash.process_options options, :task_cpus
+    exclusive  = IndiferentHash.process_options options, :exclusive
+    highmem    = IndiferentHash.process_options options, :highmem
+    licenses   = IndiferentHash.process_options options, :licenses
+    constraint = IndiferentHash.process_options options, :constraint
+    gres       = IndiferentHash.process_options options, :gres
+
+    constraint     = [constraint, "highmem"].compact * "&" if highmem
+
+    mem            = IndiferentHash.process_options options, :mem
+    mem_per_cpu    = IndiferentHash.process_options options, :mem_per_cpu
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    qsub_params = {  "-l filesystems=" => filesystems,
+                     "-l system=" => system,
+                     "-l select=" => nodes,
+                     "-l place=" => place,
+                     "-l walltime=" => time,
+                     "-q " => queue,
+                     "-A " => account,
+                     "-o " => fout,
+                     "-e " => ferr,
+                     "-k doe" => true,
+                     # "cpus-per-task" => task_cpus,
+                     # "nodes" => nodes,
+                     # "time" => time,
+                     # "constraint" => constraint,
+                     # "exclusive" => exclusive,
+                     # "licenses" => licenses,
+                     # "gres" => gres,
+                     # "mem" => mem,
+                     # "mem-per-cpu" => mem_per_cpu,
+    }
+
+
+    header =<<-EOF
+#!/bin/bash
+    EOF
+
+    qsub_params.each do |name,value|
+      next if value.nil? || value == ""
+      if TrueClass === value
+        header << "#PBS #{name}" << "\n"
+      elsif Array === value
+        value.each do |v|
+          header << "#PBS #{name}\"#{v}\"" << "\n"
+        end
+      else
+        header << "#PBS #{name}\"#{value}\"" << "\n"
+      end
+    end
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    Log.info "Issuing PBS file: #{fcmd}"
+    Log.debug Open.read(fcmd)
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
+      canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+      if normal_dep_str.nil? && canfail_dep_str.nil?
+        dep_str = ""
+      else
+        dep_str = '-W depend=' + [normal_dep_str, canfail_dep_str].compact * ","
+      end
+
+      cmd = "qsub #{dep_str} '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "squb '#{fcmd}'")
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt pbs tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "SBATCH job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("qstat").read
+    else
+      begin
+        CMD.cmd("qstat #{job}").read
+      rescue
+        ""
+      end
+    end
+  end
+
+end
+

data/lib/scout/workflow/deployment/scheduler/slurm.rb

@@ -0,0 +1,158 @@
+require_relative 'job'
+require 'scout'
+
+module SLURM 
+
+  extend SchedulerJob
+
+  def self.system
+    "SLURM"
+  end
+
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" 
+[ ! -z $SLURM_MEM_PER_NODE ] && MAX_MEMORY="$SLURM_MEM_PER_NODE"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$SLURM_JOB_ID
+export BATCH_SYSTEM=#{system}
+    EOF
+  end
+
+  def self.header(options = {})
+    options = options.dup
+
+    queue      = IndiferentHash.process_options options, :queue
+    account    = IndiferentHash.process_options options, :account
+    partition  = IndiferentHash.process_options options, :partition
+    task_cpus  = IndiferentHash.process_options options, :task_cpus
+    time       = IndiferentHash.process_options options, :time
+    nodes      = IndiferentHash.process_options options, :nodes
+    workdir    = IndiferentHash.process_options options, :workdir
+    exclusive  = IndiferentHash.process_options options, :exclusive
+    highmem    = IndiferentHash.process_options options, :highmem
+    licenses   = IndiferentHash.process_options options, :licenses
+    constraints = IndiferentHash.process_options options, :constraints
+    gres       = IndiferentHash.process_options options, :gres
+
+    constraints     = [constraints, "highmem"].compact * "&" if highmem
+
+    mem            = IndiferentHash.process_options options, :mem
+    mem_per_cpu    = IndiferentHash.process_options options, :mem_per_cpu
+
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    sbatch_params = {"job-name" => batch_name,
+                     "qos" => queue,
+                     "account" => account,
+                     "partition" => partition,
+                     "output" => fout,
+                     "error" => ferr,
+                     "cpus-per-task" => task_cpus,
+                     "nodes" => nodes,
+                     "time" => time,
+                     "constraints" => constraints,
+                     "exclusive" => exclusive,
+                     "licenses" => licenses,
+                     "gres" => gres,
+                     "mem" => mem,
+                     "mem-per-cpu" => mem_per_cpu,
+    }
+
+
+    header =<<-EOF
+#!/bin/bash
+    EOF
+
+    sbatch_params.each do |name,value|
+      next if value.nil? || value == ""
+      if TrueClass === value
+        header << "#SBATCH --#{name}" << "\n"
+      elsif Array === value
+        value.each do |v|
+          header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
+        end
+      else
+        header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
+      end
+    end
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    Log.info "Issuing SLURM file: #{fcmd}"
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
+      canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+      if normal_dep_str.nil? && canfail_dep_str.nil?
+        dep_str = ""
+      else
+        dep_str = '--dependency=' + [normal_dep_str, canfail_dep_str].compact * ","
+      end
+
+      cmd = "sbatch #{dep_str} '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "sbatch '#{fcmd}'")
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local scout): ") + Log.color(:blue, "scout slurm tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "SBATCH job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("squeue").read
+    else
+      begin
+        CMD.cmd("squeue --job #{job}").read
+      rescue
+        ""
+      end
+    end
+  end
+end

data/lib/scout/workflow/deployment/scheduler.rb

@@ -0,0 +1,73 @@
+require_relative 'orchestrator/chains'
+require_relative 'orchestrator/rules'
+require_relative 'orchestrator/batches'
+
+require_relative 'scheduler/slurm'
+require_relative 'scheduler/pbs'
+require_relative 'scheduler/lfs'
+
+module Workflow::Scheduler
+  def self.produce(jobs, rules = {}, options = {})
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    Workflow::Scheduler.process_batches(batches, options)
+  end
+
+  def self.process_batches(batches, process_options = {})
+    failed_jobs = []
+
+    pending = batches.dup
+
+    sorted = []
+    while pending.any?
+      leaf_nodes = batches.select{|batch| (batch[:deps] - sorted).empty? }
+      sorted.concat(leaf_nodes - sorted)
+      pending -= leaf_nodes
+    end
+
+    batch_system = Scout::Config.get :system, :batch, :scheduler, 'env:BATCH_SYSTEM', default: 'SLURM'
+
+    batch_ids = {}
+    sorted.collect do |batch|
+      job_options = batch[:rules]
+      job_options = IndiferentHash.add_defaults job_options, process_options.dup
+
+      if batch[:deps].nil?
+        batch_dependencies = [] 
+      else 
+        top_jobs = batch[:jobs]
+
+        batch_dependencies = batch[:deps].collect{|dep| 
+          dep_target = dep[:top_level]
+          id = batch_ids[dep_target].to_s
+
+          if dep_target.canfail?
+            'canfail:' + id
+          else
+            id
+          end
+        }
+      end
+
+      job_options.merge!(:batch_dependencies => batch_dependencies )
+      job_options.merge!(:manifest => batch[:jobs].collect{|d| d.task_signature })
+
+      begin
+        id, dir = case batch_system
+             when 'SLURM'
+               SLURM.run_job(batch[:top_level], job_options)
+             when 'LSF'
+               LSF.run_job(batch[:top_level], job_options)
+             when 'PBS'
+               PBS.run_job(batch[:top_level], job_options)
+             when nil
+               raise "No batch system specified"
+             else
+               raise "Unknown batch system #{batch_system}"
+             end
+        batch_ids[batch[:top_level]] = id
+      rescue DryRun
+        $!.message
+      end
+    end
+  end
+end

data/lib/scout/workflow/deployment.rb

@@ -1,2 +1,11 @@
-require_relative 'deployment/orchestrator'
+require_relative 'deployment/local'
+require_relative 'deployment/scheduler'
 require_relative 'deployment/trace'
+require_relative 'deployment/queue'
+
+module Workflow
+  def self.produce(jobs, ...)
+    rules = Workflow::Orchestrator.load_rules_for_job(jobs)
+    Workflow::LocalExecutor.produce(jobs, rules, ...)
+  end
+end

data/lib/scout/workflow/exceptions.rb

@@ -1 +1,3 @@
 class TaskNotFound < StandardError; end
+class DryRun < Exception; end
+

data/lib/scout/workflow/step/config.rb

@@ -12,6 +12,9 @@ class Step
       new_tokens << ("task:" + workflow_name << "#" << task_name.to_s)
     end
     new_tokens << ("task:" + task_name.to_s)
+    new_tokens << (task_name.to_s)
+    new_tokens << (workflow_name)
+    new_tokens << ("task")
 
     Scout::Config.get(key, tokens + new_tokens, options)
   end

data/lib/scout/workflow/step/info.rb

@@ -38,8 +38,8 @@ class Step
     save_info(@info = info)
   end
 
-  def init_info
-    log :waiting unless info_file.nil? || Open.exists?(info_file)
+  def init_info(status=:waiting)
+    log status unless info_file.nil? || Open.exists?(info_file)
   end
 
   def info

data/lib/scout/workflow/step/progress.rb

@@ -25,5 +25,57 @@ class Step
     kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
     TSV.traverse obj, **kwargs, &block
   end
+
+  def monitor_stream(stream, options = {}, &block)
+    case options[:bar] 
+    when TrueClass
+      bar = progress_bar 
+    when Hash
+      bar = progress_bar options[:bar]
+    when Numeric
+      bar = progress_bar :max => options[:bar]
+    else
+      bar = options[:bar]
+    end
+
+    out = if bar.nil?
+            Open.line_monitor_stream stream, &block
+          elsif (block.nil? || block.arity == 0)
+            Open.line_monitor_stream stream do
+              bar.tick
+            end
+          elsif block.arity == 1
+            Open.line_monitor_stream stream do |line|
+              bar.tick
+              block.call line
+            end
+          elsif block.arity == 2
+            Open.line_monitor_stream stream do |line|
+              block.call line, bar
+            end
+          end
+
+    if bar
+      ConcurrentStream.setup(out, :abort_callback => Proc.new{
+        bar.done
+        Log::ProgressBar.remove_bar(bar, true)
+      }, :callback => Proc.new{
+        bar.done
+        Log::ProgressBar.remove_bar(bar)
+      })
+    end
+
+    bgzip = (options[:compress] || options[:gzip]).to_s == 'bgzip'
+    bgzip = true if options[:bgzip]
+
+    gzip = true if options[:compress] || options[:gzip]
+    if bgzip
+      Open.bgzip(out)
+    elsif gzip
+      Open.gzip(out)
+    else
+      out
+    end
+  end
 end
 

data/lib/scout/workflow/step.rb

@@ -43,6 +43,18 @@ class Step
                   end
                 end
   end
+
+  def non_default_inputs
+    @non_default_inputs ||= begin
+                  if info_file && Open.exists?(info_file)
+                    info[:non_default_inputs]
+                  else
+                    []
+                  end
+                end
+  end
+
+
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
@@ -90,6 +102,8 @@ class Step
     return name.split(".").first
   end
 
+  attr_accessor :task_name
+
   def task_name
     return @task_name if @task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
@@ -189,6 +203,7 @@ class Step
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
           :provided_inputs => Annotation.purge(provided_inputs),
+          :non_default_inputs => non_default_inputs,
           :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 
@@ -223,8 +238,22 @@ class Step
 
       @result
     rescue Exception => e
-      merge_info :status => :error, :exception => Base64.encode64(Marshal.dump(e)), :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
       begin
+        begin
+          if ConcurrentStreamProcessFailed === e
+            s = e.concurrent_stream
+            e.concurrent_stream = nil
+            exception_encoded = Base64.encode64(Marshal.dump(e))
+            e.concurrent_stream = s
+          else
+            exception_encoded = Base64.encode64(Marshal.dump(e))
+          end
+          merge_info :status => :error, :exception => exception_encoded, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+        rescue Exception
+          exception_encoded = Base64.encode64(Marshal.dump(Exception.new(e.message)))
+          merge_info :status => :error, :exception => exception_encoded, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+        end
+
         abort_dependencies
       ensure
         raise e