scout-gear 9.0.0 → 10.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98b3f84350f2f451b717f004bfc97cd7767f7e0fed58e1a2d0cd023bc700185f
-  data.tar.gz: '00847ad4d92b1cab22305ea867dc12c870b22e192e79536917f82aa90cc82989'
+  metadata.gz: 5af158e978a724dacd722c9584cb4fd3b7f46fff8a4619272b1d2c2258418600
+  data.tar.gz: 6bd5da8eba38542376f4b5566154fc16105f3742ba0758a37139d0ff9679d06e
 SHA512:
-  metadata.gz: 4af5e13873fa13dea33161e9200aa725bb4329899e9242ae6504ffae2ab82e7eb01b82a53edc2a4331ede06d0f12bb07f86f579cb29ecbf0d36e791bf6d53f3e
-  data.tar.gz: fc773eb933e5c851aa1d642e872df02d282fbcb29ee1b8e1bf150ffebf9bd4ce8c3670d43a284ff2c8197477a8cf6f71dfcd49b265ad938c723292ec57579998
+  metadata.gz: e9f6f4d2a5e223d740332b627c9606907d69536696f477401087b4f72bee09d1b374d060275ecb3fad5e38c27010417d36d661d985c58d71e3cec4bf04bbcbf7
+  data.tar.gz: a7b8e5d1bfd1057bd93cfbd4a2439bccf0e28848f3e08923c81da8bcc2fe1a489ffb2e220587ecb4533ad0cf7edcf506f3c4557955dfa11356d816b87b379968

data/.vimproject

@@ -78,12 +78,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    open.rb
    open=open{
     lock.rb
+    remote.rb
+    stream.rb
+    util.rb
     lock=lock{
      lockfile.rb
     }
-    stream.rb
-    remote.rb
-    util.rb
    }
    resource.rb
    resource=resource{

data/Rakefile

@@ -20,10 +20,9 @@ Juwelier::Tasks.new do |gem|
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
   #  gem.add_development_dependency 'rspec', '> 1.2.3'
-  gem.add_runtime_dependency 'term-ansicolor'
+  gem.add_runtime_dependency 'scout-essentials'
   gem.add_runtime_dependency 'net-ssh'
   gem.add_runtime_dependency 'matrix'
-  gem.add_runtime_dependency 'sys-proctable'
   gem.add_runtime_dependency 'RubyInline'
   #gem.add_runtime_dependency 'tokyocabinet'
 

data/VERSION

@@ -1 +1 @@
-9.0.0
+10.0.1

data/bin/scout

@@ -99,7 +99,6 @@ if config_keys = options.delete(:config_keys)
   end
 end
 
-
 $scout_command_dir = Scout.bin.scout
 $scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 

data/lib/rbbt-scout.rb

@@ -1,7 +1,8 @@
 $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
-module Rbbt
+module Scout
   extend Resource
   self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end
+Rbbt = Scout
 
 Resource.set_software_env Rbbt.software

data/lib/scout/offsite/exceptions.rb

@@ -0,0 +1,9 @@
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/offsite/ssh.rb

@@ -1,4 +1,5 @@
 require 'net/ssh'
+require_relative 'exceptions'
 
 class SSHLine
   class << self

data/lib/scout/semaphore.rb

@@ -8,6 +8,8 @@ end
 
 if continue
   module ScoutSemaphore
+    class SemaphoreInterrupted < TryAgain; end
+
     inline(:C) do |builder|
       builder.prefix <<-EOF
   #include <unistd.h>

data/lib/scout/tsv/dumper.rb

@@ -19,6 +19,7 @@ module TSV
         preamble_str = nil
       end
 
+      preamble_str = preamble_str.strip if preamble_str
       [preamble_str, fields_str].compact * "\n"
     end
 

data/lib/scout/tsv/open.rb

@@ -1,4 +1,4 @@
-require_relative '../open'
+require 'scout/open'
 require_relative '../work_queue'
 
 module MultipleResult
@@ -20,7 +20,7 @@ module Open
         into.add *res
       when TSV, Hash
         key, value = res
-        if into.type == :double
+        if TSV === into && into.type == :double
           into.zip_new key, value, insitu: false
         else
           into[key] = value
@@ -118,6 +118,12 @@ module Open
                 callback.call res if callback
                 nil
               end
+            when Hash
+              obj.each do |key,value|
+                res = block.call(key,value)
+                callback.call res if callback
+                nil
+              end
             when Array
               obj.each do |line|
                 res = block.call(line)

data/lib/scout/tsv/parser.rb

@@ -1,4 +1,4 @@
-require_relative '../named_array'
+require 'scout/named_array'
 module TSV
   def self.cast_value(value, cast)
     if Array === value

data/lib/scout/tsv/persist/adapter.rb

@@ -1,5 +1,5 @@
-require_relative '../../open/lock'
-require_relative '../../meta_extension'
+require 'scout/open/lock'
+require 'scout/meta_extension'
 require_relative 'serialize'
 
 module TSVAdapter

data/lib/scout/tsv/persist.rb

@@ -1,3 +1,4 @@
+require 'scout/persist'
 require_relative 'persist/adapter'
 require_relative 'persist/tokyocabinet'
 

data/lib/scout/tsv/util.rb

@@ -122,7 +122,7 @@ Example:
   end
 
   def digest_str
-    fingerprint
+    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
   end
 
   def inspect

data/lib/scout/tsv.rb

@@ -1,4 +1,4 @@
-require_relative 'meta_extension'
+require 'scout/meta_extension'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'

data/lib/scout/work_queue/exceptions.rb

@@ -0,0 +1,18 @@
+class DoneProcessing < Exception
+  attr_accessor :pid
+  def initialize(pid = Process.pid)
+    @pid = pid
+  end
+
+  def message
+    "Done processing pid #{pid}"
+  end
+end
+
+class WorkerException < ScoutException
+  attr_accessor :worker_exception, :pid
+  def initialize(worker_exception, pid)
+    @worker_exception = worker_exception
+    @pid = pid
+  end
+end

data/lib/scout/work_queue.rb

@@ -1,5 +1,6 @@
 require_relative 'work_queue/socket'
 require_relative 'work_queue/worker'
+require_relative 'work_queue/exceptions'
 require 'timeout'
 
 class WorkQueue

data/lib/scout/workflow/definition.rb

@@ -1,4 +1,4 @@
-require_relative '../meta_extension'
+require 'scout/meta_extension'
 
 module Workflow
   extend MetaExtension

data/lib/scout/workflow/step/config.rb

@@ -1,4 +1,4 @@
-require_relative '../../config'
+require 'scout/config'
 
 class Step
   def config(key, *tokens)

data/lib/scout/workflow/step/dependencies.rb

@@ -7,11 +7,12 @@ class Step
       direct_deps << dep
     end
     seen.concat direct_deps.collect{|d| d.path }
-    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, [])); acc }
+    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
   end
 
   def recursive_inputs
-    recursive_inputs = @inputs.to_hash
+    recursive_inputs = NamedArray === @inputs ? @inputs.to_hash : {}
+    return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
       acc.merge(dep.recursive_inputs)
     end

data/lib/scout/workflow/step/file.rb

@@ -2,7 +2,11 @@ class Step
   def files_dir
     @files_dir ||= begin
                      dir = @path + ".files"
-                     @path.annotate(dir) if Path === @path
+                     if Path === @path
+                       @path.annotate(dir)
+                     else
+                       Path.setup(dir)
+                     end
                      dir.pkgdir = self
                      dir
                    end

data/lib/scout/workflow/step/info.rb

@@ -9,12 +9,18 @@ class Step
                    end
   end
 
+  def self.load_info(info_file)
+    info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(info)
+  end
+
   def load_info
-    @info = Persist.load(info_file, SERIALIZER) || {}
-    IndiferentHash.setup(@info)
+    @info = Step.load_info(info_file)
     @info_load_time = Time.now
   end
 
+
+
   def save_info(info = nil)
     Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now

data/lib/scout/workflow/step/provenance.rb

@@ -83,10 +83,25 @@ class Step
       end
     end
 
-    str << "\n"
+    str
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
+  def self.prov_indent(step, offset, input_dependencies)
+    return (" " * (offset))
+    if step.alias?
+      (" " * offset + "🡵")
+    elsif step.overriden_task
+      (" " * offset + "🡇")
+    elsif input_dependencies.include?(step)
+      (" " * offset + "┝")
+    elsif step.input_dependencies.any?
+      (" " * offset + "╰")
+    else
+      (" " * (offset+1))
+    end
+  end
+
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil, input_dependencies = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -101,7 +116,7 @@ class Step
 
     this_step_msg = prov_report_msg(status, name, path, info, input)
 
-    input_dependencies = {}
+    input_dependencies ||= {}
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
@@ -116,8 +131,10 @@ class Step
       end
     end if step.dependencies
 
-    str = ""
-    str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str = []
+
+    indent = prov_indent(step, offset, input_dependencies)
+    str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     step.dependencies.dup.tap{|l| 
       l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
@@ -126,10 +143,10 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
+        str.concat(prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep], input_dependencies.dup).split("\n"))
       else
         if expand_repeats
-          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task, input_dependencies[dep], input_dependencies.dup)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -138,13 +155,14 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
+          dep_indent = prov_indent(dep, offset+1, input_dependencies)
+          str << dep_indent + Log.color(Step.status_color(status), Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies
 
-    str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str << indent + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
-    str
+    str * "\n"
   end
 end

data/lib/scout/workflow/step/status.rb

@@ -33,6 +33,10 @@ class Step
     Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
+  def self.clean(file)
+    Step.new(file).clean
+  end
+
 
   def recursive_clean
     dependencies.each do |dep|

data/lib/scout/workflow/step.rb

@@ -1,5 +1,5 @@
-require_relative '../path'
-require_relative '../persist'
+require 'scout/path'
+require 'scout/persist'
 require_relative 'step/info'
 require_relative 'step/status'
 require_relative 'step/load'
@@ -33,7 +33,9 @@ class Step
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
-                    info[:inputs]
+                    inputs = info[:inputs]
+                    NamedArray.setup(inputs, info[:input_names]) if inputs && info[:input_names]
+                    inputs
                   else
                     []
                   end
@@ -61,6 +63,10 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def short_path
+    [workflow.to_s, task_name, name] * "/"
+  end
+
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
@@ -122,9 +128,10 @@ class Step
       begin
         Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
           clear_info
+          input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
           merge_info :status => :start, :start => Time.now,
             :pid => Process.pid, :pid_hostname => Misc.hostname, 
-            :inputs => MetaExtension.purge(inputs), :type => type,
+            :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
             :dependencies => dependencies.collect{|d| d.path }
 
           @result = exec
@@ -278,9 +285,10 @@ class Step
   end
 
   def step(task_name)
+    task_name = task_name.to_sym
     dependencies.each do |dep|
-      return dep if dep.task_name == task_name
-      return dep if dep.overriden_task == task_name
+      return dep if dep.task_name && dep.task_name.to_sym == task_name
+      return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
       rec_dep = dep.step(task_name)
       return rec_dep if rec_dep
     end
@@ -302,4 +310,8 @@ class Step
   def task_signature
     [workflow.to_s, task_name] * "#"
   end
+
+  def alias?
+    task.alias? if task
+  end
 end

data/lib/scout/workflow/task/dependencies.rb

@@ -54,6 +54,7 @@ module Task
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
         dep = Step.new dep unless Step === dep
+        dep = dep.dup
         dep.type = workflow.tasks[task].type
         dep.overriden_task = task
         dep.overriden_workflow = workflow

data/lib/scout/workflow/task/inputs.rb

@@ -1,4 +1,4 @@
-require_relative '../../named_array'
+require 'scout/named_array'
 module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
@@ -87,6 +87,8 @@ module Task
       if Path.is_filename?(value) 
         relative_file = save_file_input(value, directory)
         Open.write(input_file + ".as_file", relative_file)
+      elsif Step === value
+        Open.write(input_file + ".as_step", value.short_path)
       elsif type == :file
         relative_file = save_file_input(value, directory)
         Persist.save(relative_file, input_file, :file)
@@ -115,6 +117,9 @@ module Task
           value = Open.read(filename).strip
           value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
           inputs[name] = value
+        elsif filename.end_with?('.as_step')
+          value = Open.read(filename).strip
+          inputs[name] = Step.load value
         elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
           inputs[name] = filename
         else

data/lib/scout/workflow/task.rb

@@ -1,5 +1,5 @@
-require_relative '../meta_extension'
-require_relative '../named_array'
+require 'scout/meta_extension'
+require 'scout/named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 require_relative 'task/dependencies'
@@ -45,6 +45,21 @@ module Task
     IndiferentHash.setup(provided_inputs)
     id = DEFAULT_NAME if id.nil?
 
+
+    missing_inputs = []
+    self.inputs.each do |input,type,desc,val,options|
+      next unless options && options[:required]
+      missing_inputs << input unless provided_inputs.include?(input)
+    end if self.inputs
+
+    if missing_inputs.length == 1
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
+    end
+
+    if missing_inputs.length > 1
+      raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
+    end
+
     provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
     inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
@@ -52,7 +67,7 @@ module Task
     compute = {}
     dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+    #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     non_default_inputs.uniq!
 
@@ -95,4 +110,8 @@ module Task
       Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
+
+  def alias?
+    @extension == :dep_task
+  end
 end

data/lib/scout/workflow/usage.rb

@@ -136,6 +136,7 @@ module Workflow
 
     dep_tree = {}
     task = self.tasks[task_name]
+    raise "TaskNotFound: #{task_name}" if task.nil?
     task.deps.each do |workflow, task, options|
       next if seen.include? dep
       seen << [workflow, task, options.merge(seen_options)]

data/lib/scout/workflow.rb

@@ -6,8 +6,8 @@ require_relative 'workflow/documentation'
 require_relative 'workflow/usage'
 require_relative 'workflow/deployment'
 
-require_relative 'resource'
-require_relative 'resource/scout'
+require 'scout/resource'
+require 'scout/resource/scout'
 
 module Workflow
   class << self
@@ -27,7 +27,9 @@ module Workflow
 
   def self.require_workflow(workflow_name_orig)
     first = nil
-    workflow_name_orig.split("+").each do |workflow_name|
+    workflow_name_orig.split("+").each do |complete_workflow_name|
+      self.main = nil
+      workflow_name, *subworkflows = complete_workflow_name.split("::")
       workflow = workflow_name
       workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
       workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
@@ -40,7 +42,14 @@ module Workflow
       else
         raise "Workflow #{workflow_name} not found"
       end
-      first ||= self.main || workflows.last
+
+      current = begin
+                  Kernel.const_get(complete_workflow_name)
+                rescue
+                  self.main || workflows.last
+                end
+
+      first ||= current
     end
     first
   end

data/lib/scout-gear.rb

@@ -1,12 +1,3 @@
-require_relative 'scout/exceptions'
-require_relative 'scout/indiferent_hash'
-require_relative 'scout/tmpfile'
-require_relative 'scout/log'
-require_relative 'scout/path'
-require_relative 'scout/simple_opt'
-require_relative 'scout/resource'
-require_relative 'scout/resource/scout'
-require_relative 'scout/persist'
+require 'scout-essentials'
 require_relative 'scout/tsv'
-require_relative 'scout/config'
 require_relative 'scout/offsite'