scout-gear 8.1.0 → 9.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ca509bfb5065aabcdae73545139db0f8b04748084821e002c0475a55be5cb70
-  data.tar.gz: 6f0333ca6dd0ceee0ea9290cf3c30f7164c2f0732b76ce739e29da19a0687242
+  metadata.gz: 98b3f84350f2f451b717f004bfc97cd7767f7e0fed58e1a2d0cd023bc700185f
+  data.tar.gz: '00847ad4d92b1cab22305ea867dc12c870b22e192e79536917f82aa90cc82989'
 SHA512:
-  metadata.gz: 2cbd82d97ec0dad50e624cda26ae6a05aeba8b9e5ff07a5b4746c677b8e2ab1996378237095ba09a70f8ed8cc926d782c1677aa24d6f94784f560090d6845a2c
-  data.tar.gz: aa9529b3393016c846b578b736428b964519d2fb5c1507a6335eff651d0588a5d9191cc61cff52279c18902f2efce386415099bcd92f8736f6d67cef1d33b561
+  metadata.gz: 4af5e13873fa13dea33161e9200aa725bb4329899e9242ae6504ffae2ab82e7eb01b82a53edc2a4331ede06d0f12bb07f86f579cb29ecbf0d36e791bf6d53f3e
+  data.tar.gz: fc773eb933e5c851aa1d642e872df02d282fbcb29ee1b8e1bf150ffebf9bd4ce8c3670d43a284ff2c8197477a8cf6f71dfcd49b265ad938c723292ec57579998

data/.vimproject

@@ -10,6 +10,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   glob
   doc
   update
+  template
   offsite
   workflow=workflow{
    task
@@ -17,6 +18,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    info
    install
   }
+  resource=resource{
+   produce
+  }
  }
  lib=lib {
   scout-gear.rb
@@ -115,6 +119,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      progress.rb
      provenance.rb
      status.rb
+     inputs.rb
     }
     task.rb
     task=task{
@@ -168,6 +173,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     sync.rb
     step.rb
    }
+   hpc=hpc{
+    slurm.rb
+   }
   }
  }
  test=test {
@@ -198,7 +206,18 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     test_task.rb
     test_usage.rb
     test_util.rb
+    step=step{
+     test_dependencies.rb
+     test_info.rb
+     test_load.rb
+     test_provenance.rb
+     test_status.rb
     }
+    task=task{
+     test_dependencies.rb
+     test_inputs.rb
+    }
+   }
    indiferent_hash=indiferent_hash{
     test_case_insensitive.rb
     test_options.rb
@@ -209,6 +228,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   software=software{
    install_helpers
   }
+  templates=templates{
+   workflow.rb
+  }
  }
 
 

data/VERSION

@@ -1 +1 @@
-8.1.0
+9.0.0

data/bin/scout

@@ -54,6 +54,7 @@ if dev_dir
   end
 end
 
+require 'scout'
 
 require 'scout/simple_opt'
 

data/lib/rbbt-scout.rb

@@ -1,6 +1,7 @@
 $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
 module Rbbt
   extend Resource
-  self.pkgdir = 'rbbt'
   self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end
+
+Resource.set_software_env Rbbt.software

data/lib/scout/cmd.rb

@@ -20,17 +20,22 @@ module CMD
     end
   end
 
+
   def self.get_tool(tool)
     return tool.to_s unless TOOLS[tool]
 
     @@init_cmd_tool ||= IndiferentHash.setup({})
+
+    claim, test, block, cmd = TOOLS[tool]
+    cmd = tool.to_s if cmd.nil?
+
     if !@@init_cmd_tool[tool]
-      claim, test, block, cmd = TOOLS[tool]
+
       begin
         if test
           CMD.cmd(test + " ")
         else
-          CMD.cmd("#{tool} --help")
+          CMD.cmd("#{cmd} --help")
         end
       rescue
         if claim
@@ -43,7 +48,7 @@ module CMD
       version = nil
       ["--version", "-version", "--help", ""].each do |f|
         begin
-          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
           version = CMD.scan_version_text(version_txt, tool)
           break if version
         rescue
@@ -56,9 +61,8 @@ module CMD
       return cmd if cmd
     end
 
-    tool.to_s
+    cmd
   end
-
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
     text = Misc.fixutf8 text

data/lib/scout/indiferent_hash.rb

@@ -75,5 +75,22 @@ module IndiferentHash
     end
     clean
   end
+
+  def slice(*list)
+    ext_list = []
+    list.each do |e|
+      case e
+      when Symbol
+        ext_list << e
+        ext_list << e.to_s
+      when String
+        ext_list << e
+        ext_list << e.to_sym
+      else
+        ext_list << e
+      end
+    end
+    IndiferentHash.setup(super(*ext_list))
+  end
 end
 

data/lib/scout/log/progress/report.rb

@@ -161,6 +161,7 @@ module Log
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       info.delete_if{|k,v| v.nil?}
+      FileUtils.mkdir_p File.dirname(file) unless File.exist?(File.dirname(file))
       File.write(file, info.to_yaml)
     end
 

data/lib/scout/log.rb

@@ -51,17 +51,19 @@ module Log
 
   def self.tty_size
     @@tty_size ||= Log.ignore_stderr do
-      begin
-        IO.console.winsize.last
-      rescue Exception
-        begin
-          res = `tput li`
-          res = nil if res == ""
-          res || ENV["TTY_SIZE"] || 80
-        rescue Exception
-          ENV["TTY_SIZE"] || 80
-        end
-      end
+      size = begin
+               IO.console.winsize.last
+             rescue Exception
+               begin
+                 res = `tput li`
+                 res = nil if res == ""
+                 res || ENV["TTY_SIZE"] || 80
+               rescue Exception
+                 ENV["TTY_SIZE"] || 80
+               end
+             end
+      size = size.to_i if String === size
+      size
     end
   end
 
@@ -269,6 +271,7 @@ module Log
       line = line.sub('`',"'")
       color = :green if line =~ /workflow/
       color = :blue if line =~ /scout-/
+      color = :cyan if line =~ /rbbt-/
       if color
         Log.color color, line
       else

data/lib/scout/misc/filesystem.rb

@@ -16,11 +16,10 @@ module Misc
 
     basedir += "/" unless basedir.slice(-2,-1) == "/"
 
-    if path.index(basedir) == 0
-      return path[basedir.length..-1]
+    if path.start_with?(basedir)
+      return path.slice(basedir.length, basedir.length)
     else
       return nil
     end
   end
-
 end

data/lib/scout/offsite/ssh.rb

@@ -21,6 +21,9 @@ class SSHLine
       ch.exec 'bash -l'
     end
 
+    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
+    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
+
     @ch.on_data do |_,data|
       if m = data.match(/DONECMD: (\d+)\n/)
         @exit_status = m[1].to_i

data/lib/scout/offsite/step.rb

@@ -5,14 +5,14 @@ require_relative 'sync'
 module OffsiteStep
 
   extend MetaExtension
-  extension_attr :server, :workflow_name, :clean_id, :provided_inputs
+  extension_attr :server, :workflow_name, :clean_id, :slurm
 
   def inputs_directory
     @inputs_directory ||= begin
                             if provided_inputs && provided_inputs.any?
                               file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
                               TmpFile.with_path do |inputs_dir|
-                                task.save_inputs(inputs_dir, provided_inputs)
+                                save_inputs(inputs_dir)
                                 SSHLine.rsync(inputs_dir, file, target: server, directory: true)
                               end
                               file
@@ -68,6 +68,19 @@ job = wf.job(:#{task_name}, "#{clean_name}");
     status == :done
   end
 
+  def orchestrate_slurm
+    bundle_files = offsite_job_ssh <<~EOF
+    require 'rbbt/hpc'
+    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    job.join
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+
   def exec
     bundle_files = offsite_job_ssh <<~EOF
     job.run
@@ -78,6 +91,10 @@ job = wf.job(:#{task_name}, "#{clean_name}");
   end
 
   def run
-    exec
+    if slurm
+      orchestrate_slurm
+    else
+      exec
+    end
   end
 end

data/lib/scout/open/stream.rb

@@ -66,7 +66,6 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        Log.debug "Consume stream done #{Log.fingerprint io} -> #{Log.fingerprint(into_path ||into)}"
         last_c
       rescue Aborted
         Thread.current["exception"] = true

data/lib/scout/open.rb

@@ -20,21 +20,21 @@ module Open
     end
   end
 
-  def self.get_stream(file, mode = 'r')
+  def self.get_stream(file, mode = 'r', options = {})
     return file if Open.is_stream?(file)
     return file.stream if Open.has_stream?(file)
     file = file.find if Path === file
 
-    return Open.ssh(file) if Open.ssh?(file)
-    return Open.wget(file) if Open.remote?(file)
+    return Open.ssh(file, options) if Open.ssh?(file)
+    return Open.wget(file, options) if Open.remote?(file)
 
     File.open(file, mode)
   end
 
-  def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
+  def self.file_open(file, grep = false, mode = 'r', invert_grep = false, options = {})
     Open.mkdir File.dirname(file) if mode.include? 'w'
 
-    stream = get_stream(file, mode)
+    stream = get_stream(file, mode, options)
 
     if grep
       grep(stream, grep, invert_grep)
@@ -58,7 +58,7 @@ module Open
   def self.open(file, options = {})
     if IO === file || StringIO === file
       if block_given?
-        res = yield file
+        res = yield file, options
         file.close
         return res
       else
@@ -72,7 +72,7 @@ module Open
 
     options[:noz] = true if mode.include? "w"
 
-    io = file_open(file, options[:grep], mode, options[:invert_grep])
+    io = file_open(file, options[:grep], mode, options[:invert_grep], options)
 
     io = unzip(io)   if ((String === file and zip?(file))   and not options[:noz]) or options[:zip]
     io = gunzip(io)  if ((String === file and gzip?(file))  and not options[:noz]) or options[:gzip]
@@ -88,7 +88,7 @@ module Open
       rescue DontClose
         res = $!.payload
       rescue Exception
-        io.abort if io.respond_to? :abort
+        io.abort $! if io.respond_to? :abort
         io.join if io.respond_to? :join
         raise $!
       ensure

data/lib/scout/path.rb

@@ -32,7 +32,7 @@ module Path
     prevpath = prevpath.to_s if Symbol === prevpath
 
     subpath = File.join(prevpath.to_s, subpath) if prevpath
-    new = self.empty? ? subpath : File.join(self, subpath)
+    new = self.empty? ? subpath.dup : File.join(self, subpath)
     self.annotate(new)
     new
   end

data/lib/scout/persist/serialize.rb

@@ -104,7 +104,7 @@ module Persist
 
     if type == :binary
       content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
-      Open.open(path, :mode => 'wb') do |f|
+      Open.open(file, :mode => 'wb') do |f|
         f.puts content
       end
       content

data/lib/scout/resource/open.rb

@@ -0,0 +1,8 @@
+class << Open
+  alias _just_open open
+
+  def open(file, *args, **kwargs, &block)
+    file.produce if Path === file
+    _just_open(file, *args, **kwargs, &block)
+  end
+end

data/lib/scout/resource/path.rb

@@ -3,7 +3,7 @@ module Path
     return self if ! force && (Open.exist?(self) || @produced)
     begin
       if Resource === self.pkgdir
-        self.pkgdir.produce self
+        self.pkgdir.produce self, force
       else
         false
       end
@@ -13,7 +13,7 @@ module Path
       message = $!.message
       message = "No exception message" if message.nil? || message.empty?
       Log.warn "Error producing #{self}: #{message}"
-      false
+      raise $!
     ensure
       @produced = true
     end
@@ -33,13 +33,16 @@ module Path
   end
 
   def produce_and_find(extension = nil, *args)
-    if extension
-      found = find_with_extension(extension, *args)
-      found.exists? ? found : produce_with_extension(extension, *args)
-    else
-      found = find
-      found.exists? ? found : produce(*args)
-    end
+    found = if extension
+              found = find_with_extension(extension, *args)
+              found.exists? ? found : produce_with_extension(extension, *args)
+            else
+              found = find
+              found.exists? ? found : produce(*args)
+            end
+    raise "Not found: #{self}" unless found
+
+    found
   end
 
   def relocate

data/lib/scout/resource/software.rb

@@ -6,7 +6,7 @@ module Resource
 
   def self.install(content, name, software_dir = Path.setup('software'), &block)
     software_dir ||= Path.setup('software')
-    software_dir = software_dir.find if Path === software_dir
+    software_dir = software_dir.find(:user) if Path === software_dir
 
     content = block if block_given?
 
@@ -166,11 +166,13 @@ url='#{url}'
             value.sub!(/^['"]/,'')
             value.sub!(/['"]$/,'')
             value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
-            Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
+            Log.debug "Set variable export from .post_install: #{Log.fingerprint [var,value]*"="}"
             ENV[var] = value
           end
         end
       end
     end
   end
+
+  self.set_software_env
 end

data/lib/scout/resource.rb

@@ -2,7 +2,9 @@ require_relative 'log'
 require_relative 'path'
 require_relative 'resource/produce'
 require_relative 'resource/path'
+require_relative 'resource/open'
 require_relative 'resource/util'
+require_relative 'resource/software'
 
 module Resource
   extend MetaExtension

data/lib/scout/tsv/dumper.rb

@@ -11,7 +11,9 @@ module TSV
         fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
       end
 
-      if preamble && options.values.compact.any?
+      if String === preamble
+        preamble_str = preamble
+      elsif preamble && options.values.compact.any?
         preamble_str = "#: " << IndiferentHash.hash2string(options)
       else
         preamble_str = nil

data/lib/scout/tsv/parser.rb

@@ -51,7 +51,7 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, **kargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kargs, &block)
     begin
       bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
       bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
@@ -62,7 +62,8 @@ module TSV
       data = {} if data.nil?
       merge = false if type != :double && type != :flat
       line = first_line || stream.gets
-      while line
+      while line 
+        break if head && head <= 0
         begin
           line.chomp!
           if Proc === fix
@@ -171,6 +172,7 @@ module TSV
           stream.abort($!) if stream.respond_to?(:abort)
           raise $!
         ensure
+          head = head - 1 if head
           if stream.closed?
             line = nil
           else
@@ -185,7 +187,15 @@ module TSV
       else
         bar.remove
       end if bar
-      stream.join if stream.respond_to?(:join)
+
+      if stream.respond_to?(:join)
+        eof = begin
+                stream.eof?
+              rescue IOError
+                true
+              end
+        stream.join if eof
+      end
     end
   end
 

data/lib/scout/work_queue.rb

@@ -6,6 +6,7 @@ class WorkQueue
   attr_accessor :workers, :worker_proc, :callback
 
   def initialize(workers = 0, &block)
+    workers = workers.to_i if String === workers
     @input = WorkQueue::Socket.new
     @output = WorkQueue::Socket.new
     @workers = workers.times.collect{ Worker.new }

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -6,7 +6,7 @@ module Workflow
       return workload if job.done? && job.updated?
 
       job.dependencies.each do |dep|
-        next if dep.done? && job.updated?
+        next if dep.done? && dep.updated?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -14,7 +14,7 @@ module Workflow
       end
 
       job.input_dependencies.each do |dep|
-        next if dep.done? && job.updated?
+        next if dep.done? && dep.updated?
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
@@ -85,15 +85,15 @@ module Workflow
         candidates = workload.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? }
+          reject{|k| k.done? || k.running? }
       else
         candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
           select{|k,v| v.empty? }.
           collect{|k,v| k }.
-          reject{|k| k.done? }
+          reject{|k| k.done? || k.running? }
       end
 
-      top_level = workload.keys - workload.values.flatten
+      #top_level = workload.keys - workload.values.flatten
 
       candidates = purge_duplicates candidates
       candidates = sort_candidates candidates, rules
@@ -192,7 +192,10 @@ module Workflow
         workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
+        all_jobs.each{|job| job.clean unless job.done? && job.updated? }
+
         top_level_jobs = jobs.collect{|job| job.path }
+        failed_jobs = []
         while workload.any? 
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
@@ -203,9 +206,15 @@ module Workflow
             case 
             when (job.error? || job.aborted?)
               begin
-                if job.recoverable_error?
-                  job.clean
-                  raise TryAgain
+                if job.recoverable_error? 
+                  if failed_jobs.include?(job)
+                    Log.warn "Failed twice #{job.path} with recoverable error"
+                    next
+                  else
+                    failed_jobs << job
+                    job.clean
+                    raise TryAgain
+                  end
                 else
                   next
                 end

data/lib/scout/workflow/step/dependencies.rb

@@ -1,7 +1,13 @@
 class Step
-  def rec_dependencies
-    rec_dependencies = dependencies.dup
-    dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
+  def rec_dependencies(connected = false, seen = [])
+    direct_deps = []
+    dependencies.each do |dep|
+      next if seen.include? dep.path
+      next if connected && dep.done? && dep.updated?
+      direct_deps << dep
+    end
+    seen.concat direct_deps.collect{|d| d.path }
+    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, [])); acc }
   end
 
   def recursive_inputs

data/lib/scout/workflow/step/info.rb

@@ -127,6 +127,14 @@ class Step
     ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
+  def overriden?
+    overriden_task || overriden_workflow || dependencies.select{|d| d.overriden? }.any?
+  end
+
+  def overriden_deps
+    rec_dependencies.select{|d| d.overriden? }
+  end
+
   def exception
     info[:exception]
   end

data/lib/scout/workflow/step/inputs.rb

@@ -0,0 +1,5 @@
+class Step
+  def save_inputs(inputs_dir)
+    self.task.save_inputs(inputs_dir, provided_inputs)
+  end
+end