scout-gear 5.1.1 → 6.0.0

data/lib/scout/workflow/step.rb

@@ -1,20 +1,43 @@
 require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
+require_relative 'step/load'
 
 class Step 
 
   attr_accessor :path, :inputs, :dependencies, :task
-  def initialize(path, inputs = nil, dependencies = [], &task) 
+  def initialize(path, inputs = nil, dependencies = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @task = task
   end
 
+  def inputs
+    @inputs ||= begin
+                  if Open.exists?(info_file)
+                    info[:inputs]
+                  else
+                    []
+                  end
+                end
+  end
+
+  def dependencies
+    @dependencies ||= begin
+                        if Open.exists?(info_file)
+                          info[:dependencies].collect do |path|
+                            Step.load(path)
+                          end
+                        else
+                          []
+                        end
+                      end
+  end
+
   attr_accessor :type
   def type
-    @type ||= @task.respond_to?(:type) ? @task.type : nil
+    @type ||= @task.respond_to?(:type) ? @task.type : info[:type]
   end
 
   def name
@@ -31,14 +54,15 @@ class Step
 
   attr_reader :result
   def run
+    return @result || self.load if done?
+    dependencies.each{|dep| dep.run }
     @result = Persist.persist(name, type, :path => path) do
       begin
         merge_info :status => :start, :start => Time.now,
           :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
-          :inputs => inputs,
+          :inputs => inputs, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
-        dependencies.each{|dep| dep.run }
         @result = exec
       ensure
         if streaming?
@@ -61,6 +85,11 @@ class Step
     IO === @result || StringIO === @result
   end
 
+  def stream
+    join
+    streaming? ? @result : Open.open(path)
+  end
+
   def join
     if streaming?
       Open.consume_stream(@result, false) 
@@ -83,6 +112,13 @@ class Step
     FileUtils.rm path.find if path.exist?
   end
 
+  def recursive_clean
+    dependencies.each do |dep|
+      dep.recursive_clean
+    end
+    clean
+  end
+
   def step(task_name)
     dependencies.each do |dep|
       return dep if dep.task_name == task_name

data/lib/scout/workflow/task/inputs.rb

@@ -22,7 +22,7 @@ module Task
   end
 
   def assign_inputs(provided_inputs = {})
-    if self.inputs.nil?
+    if self.inputs.nil? || (self.inputs.empty? && Array === provided_inputs)
       case provided_inputs
       when Array
         return [provided_inputs, provided_inputs]
@@ -31,6 +31,8 @@ module Task
       end
     end
 
+    IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
+
     input_array = []
     non_default_inputs = []
     self.inputs.each_with_index do |p,i|
@@ -58,7 +60,7 @@ module Task
   end
   
   def process_inputs(provided_inputs = {})
-    input_array, non_default_inputs = assign_inputs(provided_inputs)
+    input_array, non_default_inputs = assign_inputs provided_inputs
     digest = Misc.digest(input_array)
     [input_array, non_default_inputs, digest]
   end

data/lib/scout/workflow/task.rb

@@ -16,6 +16,19 @@ module Task
     end
   end
 
+  def inputs
+    @inputs ||= []
+  end
+
+  def recursive_inputs
+    return inputs if deps.nil?
+    deps.inject(inputs) do |acc,dep|
+      workflow, task = dep
+      next if workflow.nil?
+      acc += workflow.tasks[task].recursive_inputs
+    end
+  end
+
   def directory
     @directory ||= Task.default_directory
   end
@@ -121,6 +134,7 @@ module Task
   def job(id = DEFAULT_NAME, provided_inputs = nil )
     provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
     provided_inputs = {} if provided_inputs.nil?
+    id = DEFAULT_NAME if id.nil?
 
     inputs, non_default_inputs, input_hash = process_inputs provided_inputs
 
@@ -136,6 +150,6 @@ module Task
 
     path = directory[id]
 
-    Step.new path, inputs, dependencies, &self
+    Step.new path.find, inputs, dependencies, &self
   end
 end

data/lib/scout/workflow/usage.rb

@@ -1,24 +1,40 @@
 require 'scout/simple_opt'
 
 module Task
-  def doc(workflow = nil, deps = nil)
-    puts Log.color(:yellow, "## #{ name }") << ":"
-    puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
-    puts
+  def usage(workflow = nil, deps = nil)
+    str = StringIO.new
+    str.puts Log.color(:yellow, name)
+    str.puts Log.color(:yellow, "-" * name.length)
+    str.puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
+    str.puts
 
     selects = []
     if inputs && inputs.any?
-      inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select && input_options[i] && input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]]  }
-      puts SOPT.input_doc(inputs, input_types, input_descriptions, input_defaults, true)
-      puts
+      str.puts Log.color(:magenta, "Inputs")
+      str.puts
+      str.puts SOPT.input_array_doc(inputs)
+      str.puts
     end
 
     if deps and deps.any?
-      puts Log.color(:magenta, "Inputs from dependencies:")
-      puts
-      seen = inputs
-      tasks.each do |name,task|
-        task.inputs - seen
+      seen = inputs.collect{|name,_| name }
+      dep_inputs = {}
+      deps.each do |dep_workflow,task_name|
+        task = dep_workflow.tasks[task_name]
+        next if task.inputs.nil?
+        inputs = task.inputs.reject{|name, _| seen.include? name }
+        next unless inputs.any?
+        dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
+        dep_inputs[dep] = inputs
+      end
+
+      str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
+      dep_inputs.each do |dep,inputs|
+        str.puts
+        str.puts Log.color :yellow, dep + ":"
+        str.puts
+        str.puts SOPT.input_array_doc(inputs)
+        str.puts
       end
 
       #task_inputs = dep_inputs deps, workflow
@@ -44,30 +60,49 @@ module Task
     end
 
     case
-    when (input_types.values & [:array]).any?
-      puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
+    when inputs && inputs.select{|name,type| type == :array }.any?
+      str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
       also works in addition to the others. You may use the '--array_separator' option
       the change this default. Whenever a file is specified it may also accept STDIN using
       the '-' character."))
-      puts
+      str.puts
 
-    when (input_types.values & [:text, :tsv]).any?
-      puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
-      puts
+    when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
+      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
+      str.puts
     end
 
-    puts Log.color(:magenta, "Returns: ") << Log.color(:blue, result_type.to_s) << "\n"
-    puts
+    str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
+    str.puts
 
     if selects.any?
-      puts Log.color(:magenta, "Input select options")
-      puts
+      str.puts Log.color(:magenta, "Input select options")
+      str.puts
       selects.collect{|p| p}.uniq.each do |input,options|
-        puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
-        puts unless Log.compact
+        str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
+        str.puts unless Log.compact
       end
-      puts
+      str.puts
+    end
+    str.rewind
+    str.read
+  end
+
+  def SOPT_str
+    sopt_options = []
+    self.recursive_inputs.each do |name,type,desc,default,options|
+      shortcut = (options && options[:shortcut]) || name.to_s.slice(0,1)
+      boolean = type == :boolean
+
+      sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
     end
+
+    sopt_options * ":"
+  end
+
+  def get_SOPT(task)
+    sopt_option_string = self.SOPT_str
+    SOPT.get sopt_option_string
   end
 end
 
@@ -167,29 +202,34 @@ module Workflow
     lines * "\n"
   end
 
-  def doc(task = nil, abridge = false)
+  def usage(task = nil, abridge = false)
 
-    if task.nil?
-      puts Log.color :magenta, self.to_s 
-      puts Log.color :magenta, "=" * self.to_s.length
+    str = StringIO.new
 
-      if self.documentation[:title] and not self.documentation[:title].empty?
-        puts
-        puts Misc.format_paragraph self.documentation[:title] 
-      end
+    if self.documentation[:title] and not self.documentation[:title].empty?
+      title = self.name + " - " + self.documentation[:title]
+      str.puts Log.color :magenta, title
+      str.puts Log.color :magenta, "=" * title.length
+    else
+      str.puts Log.color :magenta, self.name 
+      str.puts Log.color :magenta, "=" * self.name.length
+    end
 
-      if self.documentation[:description] and not self.documentation[:description].empty?
-        puts
-        puts Misc.format_paragraph self.documentation[:description] 
-      end
-      puts
+    if self.documentation[:description] and not self.documentation[:description].empty?
+      str.puts
+      str.puts Misc.format_paragraph self.documentation[:description] 
+      str.puts
+    end
 
-      puts Log.color :magenta, "## TASKS"
+
+    if task.nil?
+
+      str.puts Log.color :magenta, "## TASKS"
       if self.documentation[:task_description] and not self.documentation[:task_description].empty?
-        puts
-        puts Misc.format_paragraph self.documentation[:task_description] 
+        str.puts
+        str.puts Misc.format_paragraph self.documentation[:task_description] 
       end
-      puts
+      str.puts
 
       final = Set.new
       not_final = Set.new
@@ -208,10 +248,10 @@ module Workflow
         description = description.split("\n\n").first
 
         next if abridge && ! final.include?(name)
-        puts Misc.format_definition_list_item(name.to_s, description, Log.tty_size || 80, 30, :yellow)
+        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
 
         prov_string = prov_string(dep_tree(name))
-        puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
+        str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
       end
 
     else
@@ -224,56 +264,36 @@ module Workflow
       end
 
       #dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
-      task.doc(self, self.recursive_deps(task_name))
+      str.puts task.usage(self, self.recursive_deps(task_name))
 
       dep_tree = {[self, task_name] => dep_tree(task_name)}
       prov_tree = prov_tree(dep_tree)
       if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
 
-        puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
-        puts
+        str.puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
+        str.puts
         prov_tree.split("\n").each do |line|
           next if line.strip.empty?
           if m = line.match(/^( *)(\w+?)#(\w*)/i)
               offset, workflow, task_name =  m.values_at 1, 2, 3
-            puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
+              str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
           else
-            puts Log.color :blue, line 
+            str.puts Log.color :blue, line 
           end
         end
-        puts
-      end
-
-      if self.examples.include? task_name
-        self.examples[task_name].each do |example|
-
-          puts Log.color(:magenta, "Example ") << Log.color(:green, example) + " -- " + Log.color(:blue, example_dir[task_name][example])
-
-          inputs = self.example(task_name, example)
-
-          inputs.each do |input, type, file|
-            case type
-            when :tsv, :array, :text, :file
-              lines = file.read.split("\n")
-              head = lines[0..5].compact * "\n\n"
-              head = head[0..500]
-              puts Misc.format_definition_list_item(input, head, 1000, -1, :blue).gsub(/\n\s*\n/,"\n") 
-              puts '...' if lines.length > 6
-            else
-              puts Misc.format_definition_list_item(input, file.read, Log.tty_size, 20, :blue)
-            end
-          end
-          puts
-        end
+        str.puts
       end
     end
+
+    str.rewind
+    str.read
   end
 
   def SOPT_str(task)
     sopt_options = []
-    self.rec_inputs(task.name).each do |name|
-      short = name.to_s.chars.first
-      boolean = self.rec_input_types(task.name)[name].to_sym == :boolean
+    self.tasks[task].recursive_inputs.each do |name,type,desc,default,options|
+      shortcut = options[:shortcut] || name.to_s.slice(0,1)
+      boolean = type == :boolean
 
       sopt_options << "-#{short}--#{name}#{boolean ? "" : "*"}"
     end

data/lib/scout/workflow.rb

@@ -25,6 +25,7 @@ module Workflow
   end
 
   def self.require_workflow(workflow)
+    workflow = Path.setup('workflows')[workflow]["workflow.rb"] unless Open.exists?(workflow)
     if Open.exists?(workflow)
       workflow = workflow.find if Path === workflow
       load workflow

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 5.1.1 ruby lib
+# stub: scout-gear 6.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "5.1.1"
+  s.version = "6.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-04-27"
+  s.date = "2023-04-29"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -49,6 +49,7 @@ Gem::Specification.new do |s|
     "lib/scout/misc/filesystem.rb",
     "lib/scout/misc/format.rb",
     "lib/scout/misc/insist.rb",
+    "lib/scout/misc/monitor.rb",
     "lib/scout/open.rb",
     "lib/scout/open/lock.rb",
     "lib/scout/open/remote.rb",
@@ -67,6 +68,8 @@ Gem::Specification.new do |s|
     "lib/scout/resource/produce.rb",
     "lib/scout/resource/produce/rake.rb",
     "lib/scout/resource/scout.rb",
+    "lib/scout/resource/util.rb",
+    "lib/scout/semaphore.rb",
     "lib/scout/simple_opt.rb",
     "lib/scout/simple_opt/accessor.rb",
     "lib/scout/simple_opt/doc.rb",
@@ -74,11 +77,15 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/parse.rb",
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
+    "lib/scout/work_queue.rb",
+    "lib/scout/work_queue/socket.rb",
+    "lib/scout/work_queue/worker.rb",
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/info.rb",
+    "lib/scout/workflow/step/load.rb",
     "lib/scout/workflow/task.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
@@ -89,9 +96,15 @@ Gem::Specification.new do |s|
     "scout_commands/find",
     "scout_commands/glob",
     "scout_commands/rbbt",
+    "scout_commands/workflow/info",
+    "scout_commands/workflow/list",
     "scout_commands/workflow/task",
+    "scout_commands/workflow/task_old",
+    "share/color/color_names",
+    "share/color/diverging_colors.hex",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
+    "test/scout/log/test_color.rb",
     "test/scout/log/test_progress.rb",
     "test/scout/misc/test_digest.rb",
     "test/scout/misc/test_filesystem.rb",
@@ -107,6 +120,8 @@ Gem::Specification.new do |s|
     "test/scout/persist/test_serialize.rb",
     "test/scout/resource/test_path.rb",
     "test/scout/resource/test_produce.rb",
+    "test/scout/resource/test_util.rb",
+    "test/scout/simple_opt/test_doc.rb",
     "test/scout/simple_opt/test_get.rb",
     "test/scout/simple_opt/test_parse.rb",
     "test/scout/simple_opt/test_setup.rb",
@@ -120,10 +135,17 @@ Gem::Specification.new do |s|
     "test/scout/test_path.rb",
     "test/scout/test_persist.rb",
     "test/scout/test_resource.rb",
+    "test/scout/test_semaphore.rb",
     "test/scout/test_tmpfile.rb",
+    "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
+    "test/scout/work_queue/test_socket.rb",
+    "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/step/test_info.rb",
+    "test/scout/workflow/step/test_load.rb",
     "test/scout/workflow/task/test_inputs.rb",
+    "test/scout/workflow/test_definition.rb",
+    "test/scout/workflow/test_documentation.rb",
     "test/scout/workflow/test_step.rb",
     "test/scout/workflow/test_task.rb",
     "test/scout/workflow/test_usage.rb",

data/scout_commands/workflow/info

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Show info from job
+
+$ #{$0} [<options>] <step_path>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+path = ARGV.first
+raise MissingParameterException.new :step_path if path.nil?
+step = Step.load(path)
+
+puts step.info.to_json
+

data/scout_commands/workflow/list

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+List all workflows
+
+$ #{$0} [<options>] 
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+Path.setup('workflows').glob_all("*").each do |file|
+  puts File.basename(file)
+end
+