scout-gear 10.8.4 → 10.10.1

data/lib/scout/workflow/step/progress.rb

@@ -25,5 +25,57 @@ class Step
     kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
     TSV.traverse obj, **kwargs, &block
   end
+
+  def monitor_stream(stream, options = {}, &block)
+    case options[:bar] 
+    when TrueClass
+      bar = progress_bar 
+    when Hash
+      bar = progress_bar options[:bar]
+    when Numeric
+      bar = progress_bar :max => options[:bar]
+    else
+      bar = options[:bar]
+    end
+
+    out = if bar.nil?
+            Open.line_monitor_stream stream, &block
+          elsif (block.nil? || block.arity == 0)
+            Open.line_monitor_stream stream do
+              bar.tick
+            end
+          elsif block.arity == 1
+            Open.line_monitor_stream stream do |line|
+              bar.tick
+              block.call line
+            end
+          elsif block.arity == 2
+            Open.line_monitor_stream stream do |line|
+              block.call line, bar
+            end
+          end
+
+    if bar
+      ConcurrentStream.setup(out, :abort_callback => Proc.new{
+        bar.done
+        Log::ProgressBar.remove_bar(bar, true)
+      }, :callback => Proc.new{
+        bar.done
+        Log::ProgressBar.remove_bar(bar)
+      })
+    end
+
+    bgzip = (options[:compress] || options[:gzip]).to_s == 'bgzip'
+    bgzip = true if options[:bgzip]
+
+    gzip = true if options[:compress] || options[:gzip]
+    if bgzip
+      Open.bgzip(out)
+    elsif gzip
+      Open.gzip(out)
+    else
+      out
+    end
+  end
 end
 

data/lib/scout/workflow/step.rb

@@ -16,7 +16,7 @@ require_relative 'step/archive'
 class Step 
 
   attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
@@ -43,6 +43,18 @@ class Step
                   end
                 end
   end
+
+  def non_default_inputs
+    @non_default_inputs ||= begin
+                  if info_file && Open.exists?(info_file)
+                    info[:non_default_inputs]
+                  else
+                    []
+                  end
+                end
+  end
+
+
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
@@ -90,6 +102,8 @@ class Step
     return name.split(".").first
   end
 
+  attr_accessor :task_name
+
   def task_name
     return @task_name if @task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
@@ -112,17 +126,22 @@ class Step
 
   def exec
 
-    if inputs 
+    if inputs
       if Task === task
         types = task.inputs.collect{|name,type| type }
-        new_inputs = inputs.zip(types).collect{|input,info|  
+        new_inputs = inputs.zip(types).collect{|input,info|
           type, desc, default, options = info
           next input unless Step === input
           input.join if input.streaming?
           Task.format_input(input.join.path, type, options)
         }
       else
-        new_inputs = inputs.collect{|input|  
+        if Hash === inputs
+          new_inputs = inputs.values
+        else
+          new_inputs = inputs
+        end
+        new_inputs = new_inputs.collect{|input|
           Step === input ? input.load : input
         }
       end
@@ -184,6 +203,7 @@ class Step
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
           :provided_inputs => Annotation.purge(provided_inputs),
+          :non_default_inputs => non_default_inputs,
           :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 
@@ -218,8 +238,22 @@ class Step
 
       @result
     rescue Exception => e
-      merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
       begin
+        begin
+          if ConcurrentStreamProcessFailed === e
+            s = e.concurrent_stream
+            e.concurrent_stream = nil
+            exception_encoded = Base64.encode64(Marshal.dump(e))
+            e.concurrent_stream = s
+          else
+            exception_encoded = Base64.encode64(Marshal.dump(e))
+          end
+          merge_info :status => :error, :exception => exception_encoded, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+        rescue Exception
+          exception_encoded = Base64.encode64(Marshal.dump(Exception.new(e.message)))
+          merge_info :status => :error, :exception => exception_encoded, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+        end
+
         abort_dependencies
       ensure
         raise e

data/lib/scout/workflow/task/inputs.rb

@@ -169,7 +169,7 @@ module Task
   end
 
   def load_inputs(directory)
-    if Open.exists?(directory) && ! Open.directory?(directory)
+    if Open.exists?(directory) && ! Open.directory?(directory) && ! Open.size(directory) == 0
       TmpFile.with_file do |tmp_directory|
         Misc.in_dir tmp_directory do
           CMD.cmd("tar xvfz '#{directory}'")

data/lib/scout/workflow/task.rb

@@ -42,6 +42,8 @@ module Task
       memory_inputs = provided_inputs
     end
 
+    memory_inputs = nil if Array === memory_inputs && memory_inputs.compact.empty?
+    memory_inputs = nil if Hash === memory_inputs && memory_inputs.empty?
     Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?

data/lib/scout/workflow/usage.rb

@@ -35,6 +35,7 @@ module Task
       str.puts SOPT.input_array_doc(inputs)
 
       inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
+        next unless options
         selects << [name, options[:select_options]] if options[:select_options]
       end
     end
@@ -194,9 +195,9 @@ module Workflow
       if child
         #·description << "->" << task_name.to_s
       elsif first
-        description << "" << task_name.to_s
+        description += "" + task_name.to_s
       else
-        description << ";" << task_name.to_s
+        description += ";" + task_name.to_s
       end
       
       seen << [workflow, task_name]

data/lib/scout/workflow/util.rb

@@ -16,5 +16,27 @@ module Workflow
     name = name.to_sym
     dependencies.select{|dep| dep.task_name.to_sym == name }
   end
+
+  def all_tasks
+    tasks.keys
+  end
+
+  def self.list
+    Path.setup('workflows').glob('*').collect{|p| p.basename }
+  end
+
+  def task_jobs_files(task_name)
+    self.directory[task_name].glob("**").
+      collect{|f| %w(info files).include?(f.get_extension) ? f.unset_extension : f }.
+      uniq
+  end
+
+  def task_jobs(task_name)
+    task_jobs_files(task_name).collect{|f| Step.load f }
+  end
+
+  def load_job(task_name, name)
+    Step.new self.directory[task_name][name]
+  end
 end
 

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.8.4 ruby lib
+# stub: scout-gear 10.10.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.8.4".freeze
+  s.version = "10.10.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
     "LICENSE.txt",
+    "README.md",
     "README.rdoc"
   ]
   s.files = [
@@ -24,12 +25,19 @@ Gem::Specification.new do |s|
     ".gitmodules",
     ".vimproject",
     "LICENSE.txt",
+    "README.md",
     "README.rdoc",
     "Rakefile",
     "VERSION",
     "bin/scout",
-    "doc/lib/scout/path.md",
-    "doc/lib/scout/workflow/task.md",
+    "doc/Association.md",
+    "doc/Entity.md",
+    "doc/KnowledgeBase.md",
+    "doc/Persist.md",
+    "doc/Semaphore.md",
+    "doc/TSV.md",
+    "doc/WorkQueue.md",
+    "doc/Workflow.md",
     "lib/scout-gear.rb",
     "lib/scout.rb",
     "lib/scout/association.rb",
@@ -98,8 +106,17 @@ Gem::Specification.new do |s|
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/deployment.rb",
-    "lib/scout/workflow/deployment/orchestrator.rb",
+    "lib/scout/workflow/deployment/local.rb",
+    "lib/scout/workflow/deployment/orchestrator/batches.rb",
+    "lib/scout/workflow/deployment/orchestrator/chains.rb",
+    "lib/scout/workflow/deployment/orchestrator/rules.rb",
+    "lib/scout/workflow/deployment/orchestrator/workload.rb",
     "lib/scout/workflow/deployment/queue.rb",
+    "lib/scout/workflow/deployment/scheduler.rb",
+    "lib/scout/workflow/deployment/scheduler/job.rb",
+    "lib/scout/workflow/deployment/scheduler/lfs.rb",
+    "lib/scout/workflow/deployment/scheduler/pbs.rb",
+    "lib/scout/workflow/deployment/scheduler/slurm.rb",
     "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/entity.rb",
@@ -130,7 +147,9 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/cat",
     "scout_commands/doc",
+    "scout_commands/entity",
     "scout_commands/find",
     "scout_commands/glob",
     "scout_commands/kb/config",
@@ -145,6 +164,7 @@ Gem::Specification.new do |s|
     "scout_commands/resource/produce",
     "scout_commands/resource/sync",
     "scout_commands/system/clean",
+    "scout_commands/system/status",
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
@@ -226,13 +246,23 @@ Gem::Specification.new do |s|
     "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
-    "test/scout/workflow/deployment/test_orchestrator.rb",
+    "test/scout/workflow/deployment/orchestrator/test_batches.rb",
+    "test/scout/workflow/deployment/orchestrator/test_chains.rb",
+    "test/scout/workflow/deployment/orchestrator/test_rules.rb",
+    "test/scout/workflow/deployment/orchestrator/test_workload.rb",
+    "test/scout/workflow/deployment/scheduler/test_job.rb",
+    "test/scout/workflow/deployment/scheduler/test_lfs.rb",
+    "test/scout/workflow/deployment/scheduler/test_pbs.rb",
+    "test/scout/workflow/deployment/scheduler/test_slurm.rb",
+    "test/scout/workflow/deployment/test_local.rb",
+    "test/scout/workflow/deployment/test_scheduler.rb",
     "test/scout/workflow/deployment/test_trace.rb",
     "test/scout/workflow/step/test_archive.rb",
     "test/scout/workflow/step/test_children.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
+    "test/scout/workflow/step/test_progress.rb",
     "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/step/test_status.rb",
     "test/scout/workflow/task/test_dependencies.rb",
@@ -252,7 +282,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.7".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/batch/list

@@ -334,7 +334,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
-      (step.rec_dependencies + [step]).reverse.each do |j|
+      (step.rec_dependencies.to_a + [step]).reverse.each do |j|
         next if j.done? 
         if j.file(:progress).exists?
           bar = Log::ProgressBar.new 

data/scout_commands/cat

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+require 'scout-gear'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Cat a file
+
+$ #{$0} [<options>] (<resource> <path>|<path>)
+
+-h--help Print this help
+-r--requires* Files to require; 'all' for all in Scout.etc.requires
+-lw--load_workflow* Files to require; 'all' for all in Scout.etc.requires
+-w--where* Where to look for the path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+resource, path = ARGV
+path, resource = resource, nil if path.nil?
+
+raise MissingParameterException.new(:path) if path.nil?
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+location = nil
+where = options[:where]
+all = options[:all]
+
+begin
+  resource = Workflow.require_workflow resource
+rescue
+  begin
+    resource = Kernel.const_get(resource)
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end if resource
+
+path = (resource || Scout)[path.dup]
+
+if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
+  location = path.find(where)
+
+  if Array === location
+    file = location.first
+    puts location.first.read
+    puts location * "\n"
+  else
+    file = Open.exists?(location)? location : nil
+  end
+
+  puts file.read if file
+else
+  raise ParameterException, "Where '#{where}' not identified. Try scout-camp if looking for a remote file"
+end

data/scout_commands/doc

@@ -26,7 +26,9 @@ end
 
 module_name = ARGV.first
 if module_name.nil?
-  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| 
+    f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
+  } * "\n"
 else
   begin
     puts Scout.doc.lib.scout[module_name].find_with_extension('md', :lib).read

data/scout_commands/entity

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Execute an entity property
+
+$ #{$0} [<options>] <entity_type> <property> <entity> [<entity_options>] [<property_parameters>]*
+
+Property options are an array of strings that get passed to the property, the
+remaining ARGV elements; property parameters may be optional
+
+-h--help Print this help
+-W--workflows* Workflows to load
+EOF
+
+type, property, entity = ARGV
+
+if options[:help] and type.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise MissingParameterException, :type if type.nil?
+
+workflows = IndiferentHash.process_options options, :workflows, workflows: 'local'
+
+workflows = workflows.split(",")
+
+workflows.each do |workflow|
+  workflow = workflow.strip
+  wf = case workflow
+       when 'local'
+         [:lib, :current].each do |location|
+           Workflow.require_workflow Scout.root['workflow.rb'].find(location) if Scout.root['workflow.rb'].find(location).exists?
+         end
+       when 'lib'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:lib) if Scout.root['workflow.rb'].find(:lib).exists?
+       when 'current'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:current) if Scout.root['workflow.rb'].find(:current).exists?
+       else
+         Workflow.require_workflow workflow
+       end
+
+  begin
+    require "#{wf.to_s}/entity/#{Misc.snake_case(type)}"
+  rescue Exception
+  end
+
+end
+
+begin
+  require "entity/#{Misc.snake_case(type)}"
+rescue Exception
+end
+entity_module = Kernel.const_get type
+
+entity_module.annotations.uniq.each do |attr|
+  SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute" 
+end
+
+if options[:help] and property.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  properties = entity_module.properties
+
+  puts Log.color :title, 'Properties: '
+  puts properties.keys * "\n"
+
+  exit 0
+end
+
+SOPT.synopsys[' [<entity_options>]'] = ""
+
+raise MissingParameterException, :property if property.nil?
+
+
+raise ParameterException, "Property no available" unless entity_module.instance_methods.include? property.to_sym
+
+if options[:help]
+  properties = entity_module.properties[property.to_sym]
+
+  property_str = properties.collect do |type, name|
+    if type != :opt
+      "[<#{name} (#{type})>]"
+    else
+      "[<#{name}>]"
+    end
+  end * " "
+
+  SOPT.synopsys['[<property_parameters>]*'] = property_str
+  
+  if defined? scout_usage
+    scout_usage
+  else
+    puts SOPT.doc
+  end
+
+  exit 0
+end
+
+raise MissingParameterException, :entity if entity.nil?
+
+entity = Entity.prepare_entity(entity, type, options)
+
+entity_params = ARGV[3..-1]
+
+entity_params = entity_params.collect do |param|
+  if Path.located?(param) && Path.can_read?(param)
+    Open.read(param)
+  else
+    case param
+    when 'true'
+      true
+    when 'false'
+      true
+    when /^-?\d+$/
+      param.to_i
+    when /^-?\d+\.\d+$/
+      param.to_f
+    else
+      param
+    end
+  end
+end
+
+res = entity.send(property, *entity_params)
+
+case res
+when nil
+when Array
+  puts res * "\n"
+when String
+  puts res unless res.empty?
+when Symbol, Numeric
+  puts res.to_s
+when Step
+  puts res.path
+else
+  puts res.to_s
+end