scout-gear 8.0.0 → 9.0.0

data/lib/scout/workflow/usage.rb

@@ -3,17 +3,40 @@ require 'scout/simple_opt'
 module Task
   def usage(workflow = nil, deps = nil)
     str = StringIO.new
-    str.puts Log.color(:yellow, name)
-    str.puts Log.color(:yellow, "-" * name.length)
-    str.puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
-    str.puts
+
+    if description
+      title, paragraph = description.split("\n\n")
+      if title.length < Misc::MAX_TTY_LINE_WIDTH
+        title = self.name.to_s + " - " + title
+        str.puts Log.color :yellow, title
+        str.puts Log.color :yellow, "-" * title.length
+        if paragraph
+          str.puts "\n" << Misc.format_paragraph(paragraph) 
+        end
+      else
+        title = self.name.to_s
+        str.puts Log.color :yellow, title
+        str.puts Log.color :yellow, "-" * title.length
+        str.puts "\n" << Misc.format_paragraph(description) 
+      end
+    else
+      title = self.name.to_s
+      str.puts Log.color :yellow, title
+      str.puts Log.color :yellow, "-" * title.length
+    end
+
 
     selects = []
+
     if inputs && inputs.any?
+      str.puts
       str.puts Log.color(:magenta, "Inputs")
       str.puts
       str.puts SOPT.input_array_doc(inputs)
-      str.puts
+
+      inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
+        selects << [name, options[:select_options]] if options[:select_options]
+      end
     end
 
     deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
@@ -23,14 +46,21 @@ module Task
       deps.each do |dep_workflow,task_name,options|
         next if task_name.nil?
         task = dep_workflow.tasks[task_name]
+
         next if task.inputs.nil?
+
         inputs = task.inputs.reject{|name, _| seen.include? name }
         inputs = task.inputs.reject{|name, _| options.include? name }
         next unless inputs.any?
+        task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
+          selects << [i, options[:select_options]] if options[:select_options]
+        end
+
         dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
         dep_inputs[dep] = inputs
       end
 
+      str.puts
       str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
       dep_inputs.each do |dep,inputs|
         str.puts
@@ -38,54 +68,31 @@ module Task
         str.puts
         str.puts SOPT.input_array_doc(inputs)
       end
-
-      #task_inputs = dep_inputs deps, workflow
-      #task_inputs.each do |task,new_inputs|
-      #  new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t| 
-      #    t.to_sym == :select and task.input_options[i][:select_options] 
-      #  end.each do |i,t| 
-      #    selects << [i, task.input_options[i][:select_options]] 
-      #  end
-
-      #  next if new_inputs.empty?
-
-      #  if task.workflow and task.workflow != workflow
-      #    puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
-      #  else
-      #    puts "  #{Log.color :yellow, task.name.to_s}:"
-      #  end
-      #  puts unless Log.compact
-      #  puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
-      #  puts unless Log.compact
-      #end
-      #puts
     end
 
     case
     when inputs && inputs.select{|name,type| type == :array }.any?
+      str.puts
       str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
       also works in addition to the others. You may use the '--array_separator' option
       the change this default. Whenever a file is specified it may also accept STDIN using
       the '-' character."))
-      str.puts
 
     when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
-      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
       str.puts
+      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
     end
 
     str.puts
     str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
-    str.puts
 
     if selects.any?
-      str.puts Log.color(:magenta, "Input select options")
       str.puts
+      str.puts Log.color(:magenta, "Input select options")
       selects.collect{|p| p}.uniq.each do |input,options|
+        str.puts 
         str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
-        str.puts unless Log.compact
       end
-      str.puts
     end
     str.rewind
     str.read
@@ -103,9 +110,16 @@ module Task
     sopt_options * ":"
   end
 
-  def get_SOPT(task)
+  def get_SOPT
     sopt_option_string = self.SOPT_str
-    SOPT.get sopt_option_string
+    job_options = SOPT.get sopt_option_string
+    recursive_inputs.uniq.each do |name,type|
+      next unless type.to_s.include?('array')
+      if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
+        job_options[name] = job_options[name].split(",")
+      end
+    end
+    job_options
   end
 end
 
@@ -219,14 +233,16 @@ module Workflow
       str.puts Log.color :magenta, "=" * self.name.length
     end
 
-    if self.documentation[:description] and not self.documentation[:description].empty?
-      str.puts
-      str.puts Misc.format_paragraph self.documentation[:description] 
-      str.puts
-    end
+    str.puts
 
+    if tasks.nil?
+      str.puts Log.color(:title, "No tasks")
+    elsif task.nil?
 
-    if task.nil?
+      if self.documentation[:description] and not self.documentation[:description].empty?
+        str.puts Misc.format_paragraph self.documentation[:description] 
+        str.puts
+      end
 
       str.puts Log.color :magenta, "## TASKS"
       if self.documentation[:task_description] and not self.documentation[:task_description].empty?
@@ -256,7 +272,7 @@ module Workflow
 
         prov_string = prov_string(dep_tree(name))
         str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
-      end
+      end 
 
     else
 

data/lib/scout/workflow.rb

@@ -4,13 +4,14 @@ require_relative 'workflow/task'
 require_relative 'workflow/step'
 require_relative 'workflow/documentation'
 require_relative 'workflow/usage'
+require_relative 'workflow/deployment'
 
 require_relative 'resource'
 require_relative 'resource/scout'
 
 module Workflow
   class << self
-    attr_accessor :workflows
+    attr_accessor :workflows, :main
     def workflows
       @workflows ||= []
     end
@@ -24,19 +25,24 @@ module Workflow
     base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
   end
 
-  def self.require_workflow(workflow_name)
-    workflow = workflow_name 
-    workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
-    workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-    workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-    if Open.exists?(workflow)
-      workflow = workflow.find if Path === workflow
-      $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
-      load workflow
-    else
-      raise "Workflow #{workflow_name} not found"
+  def self.require_workflow(workflow_name_orig)
+    first = nil
+    workflow_name_orig.split("+").each do |workflow_name|
+      workflow = workflow_name
+      workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
+      workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
+      workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
+      if Open.exists?(workflow)
+        self.main = nil
+        workflow = workflow.find if Path === workflow
+        $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
+        load workflow
+      else
+        raise "Workflow #{workflow_name} not found"
+      end
+      first ||= self.main || workflows.last
     end
-    workflows.last
+    first
   end
 
   def job(name, *args)

data/lib/scout-gear.rb

@@ -8,3 +8,5 @@ require_relative 'scout/resource'
 require_relative 'scout/resource/scout'
 require_relative 'scout/persist'
 require_relative 'scout/tsv'
+require_relative 'scout/config'
+require_relative 'scout/offsite'

data/lib/scout.rb

@@ -1,2 +1,8 @@
 require 'workflow-scout'
 require 'rbbt-scout'
+
+module Scout
+  def self.version
+    Open.read(File.join(__dir__, '../VERSION'))
+  end
+end

data/scout-gear.gemspec

@@ -2,17 +2,17 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 8.0.0 ruby lib
+# stub: scout-gear 9.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "8.0.0"
+  s.version = "9.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-05-23"
-  s.description = "Temporary files, logs, etc.".freeze
+  s.date = "2023-06-27"
+  s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
@@ -28,6 +28,8 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/scout",
+    "doc/lib/scout/path.md",
+    "doc/lib/scout/workflow/task.md",
     "lib/rbbt-scout.rb",
     "lib/scout-gear.rb",
     "lib/scout.rb",
@@ -56,6 +58,10 @@ Gem::Specification.new do |s|
     "lib/scout/misc/monitor.rb",
     "lib/scout/misc/system.rb",
     "lib/scout/named_array.rb",
+    "lib/scout/offsite.rb",
+    "lib/scout/offsite/ssh.rb",
+    "lib/scout/offsite/step.rb",
+    "lib/scout/offsite/sync.rb",
     "lib/scout/open.rb",
     "lib/scout/open/lock.rb",
     "lib/scout/open/lock/lockfile.rb",
@@ -71,6 +77,7 @@ Gem::Specification.new do |s|
     "lib/scout/persist/path.rb",
     "lib/scout/persist/serialize.rb",
     "lib/scout/resource.rb",
+    "lib/scout/resource/open.rb",
     "lib/scout/resource/path.rb",
     "lib/scout/resource/produce.rb",
     "lib/scout/resource/produce/rake.rb",
@@ -98,6 +105,7 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/persist/fix_width_table.rb",
     "lib/scout/tsv/persist/serialize.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
+    "lib/scout/tsv/stream.rb",
     "lib/scout/tsv/transformer.rb",
     "lib/scout/tsv/traverse.rb",
     "lib/scout/tsv/util.rb",
@@ -111,33 +119,44 @@ Gem::Specification.new do |s|
     "lib/scout/work_queue/worker.rb",
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
+    "lib/scout/workflow/deployment.rb",
+    "lib/scout/workflow/deployment/orchestrator.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/config.rb",
     "lib/scout/workflow/step/dependencies.rb",
     "lib/scout/workflow/step/file.rb",
     "lib/scout/workflow/step/info.rb",
+    "lib/scout/workflow/step/inputs.rb",
     "lib/scout/workflow/step/load.rb",
     "lib/scout/workflow/step/progress.rb",
     "lib/scout/workflow/step/provenance.rb",
     "lib/scout/workflow/step/status.rb",
     "lib/scout/workflow/task.rb",
+    "lib/scout/workflow/task/dependencies.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
     "lib/scout/workflow/util.rb",
     "lib/workflow-scout.rb",
     "scout-gear.gemspec",
     "scout_commands/alias",
+    "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
+    "scout_commands/offsite",
     "scout_commands/rbbt",
+    "scout_commands/resource/produce",
+    "scout_commands/template",
+    "scout_commands/update",
     "scout_commands/workflow/info",
+    "scout_commands/workflow/install",
     "scout_commands/workflow/list",
     "scout_commands/workflow/task",
-    "scout_commands/workflow/task_old",
     "share/color/color_names",
     "share/color/diverging_colors.hex",
     "share/software/install_helpers",
+    "share/templates/command",
+    "share/templates/workflow.rb",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
     "test/scout/log/test_color.rb",
@@ -146,6 +165,10 @@ Gem::Specification.new do |s|
     "test/scout/misc/test_filesystem.rb",
     "test/scout/misc/test_insist.rb",
     "test/scout/misc/test_system.rb",
+    "test/scout/offsite/test_ssh.rb",
+    "test/scout/offsite/test_step.rb",
+    "test/scout/offsite/test_sync.rb",
+    "test/scout/offsite/test_task.rb",
     "test/scout/open/test_lock.rb",
     "test/scout/open/test_remote.rb",
     "test/scout/open/test_stream.rb",
@@ -190,6 +213,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
     "test/scout/tsv/test_persist.rb",
+    "test/scout/tsv/test_stream.rb",
     "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
     "test/scout/tsv/test_util.rb",
@@ -200,11 +224,13 @@ Gem::Specification.new do |s|
     "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
+    "test/scout/workflow/deployment/test_orchestrator.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
     "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/step/test_status.rb",
+    "test/scout/workflow/task/test_dependencies.rb",
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
@@ -213,16 +239,21 @@ Gem::Specification.new do |s|
     "test/scout/workflow/test_usage.rb",
     "test/scout/workflow/test_util.rb",
     "test/test_helper.rb",
-    "test/test_scout-gear.rb"
+    "test/test_scout-gear.rb",
+    "test/test_scout.rb"
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.4.11".freeze
+  s.rubygems_version = "3.4.13".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4
 
   s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<sys-proctable>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
   s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
   s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])

data/scout_commands/doc

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Print documentation for a module
+
+$ #{$0} [<options>] [module]
+
+If you don't specify the module it prints those available
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+module_name = ARGV.first
+if module_name.nil?
+  puts Rbbt.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+else
+  begin
+    puts Rbbt.doc.lib.scout[module_name].find_with_extension('md', :lib).read
+  rescue
+    puts Rbbt.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
+  end
+end
+

data/scout_commands/find

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'scout-gear'
+require 'scout/offsite'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 

data/scout_commands/offsite

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/offsite'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
+
+Run a command offsite. Use -- to make sure options reach the offsite
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+where = ARGV.shift
+command = ARGV.shift
+
+puts SSHLine.command(where, command, ARGV)

data/scout_commands/resource/produce

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Produce a resource
+
+$ #{$0} [<options>] <Resource> <file> 
+
+-h--help Print this help
+-W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
+-r--requires* Files to require; 'all' for all in Scout.etc.requires:
+-f--force Force the production if the file is already present
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+resource, path = ARGV
+
+begin
+  resource = Kernel.const_get(resource)
+rescue
+  begin
+    resource = Workflow.require_workflow resource
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end
+
+force = options[:force]
+
+puts resource[path].produce(force).find
+

data/scout_commands/template

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Print a template
+
+$ #{$0} [<options>] <template> [<VAR>=<value>]*
+
+Print a template making the substitutions that follow
+
+-h--help Print this help
+-s--sub* List a substitutions as a parameter instead
+-s--var* List a substitutions as a parameter, variable name
+-s--value* List a substitutions as a parameter, value
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+template, *subs = ARGV
+
+raise MissingParameterException, :template if template.nil?
+
+txt = Scout.share.templates[template].find_with_extension('rb').read
+
+if options[:sub]
+  subs ||= []
+  subs << options[:sub]
+end
+
+if options[:var]
+  subs ||= []
+  subs << [options[:var], options[:value]] * "="
+end
+
+
+subs.each do |sub|
+  key, value = sub.split("=")
+
+  txt.gsub!(key, value)
+end
+
+puts txt

data/scout_commands/update

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Update scout installation
+
+$ #{$0} [<options>] [gem]
+
+If no gem is specified 'scout-gear' is selected.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+gem_name = ARGV.first
+Misc.update_git(gem_name)
+
+

data/scout_commands/workflow/info

@@ -23,7 +23,19 @@ end
 
 path = ARGV.first
 raise MissingParameterException.new :step_path if path.nil?
-step = Step.load(path)
-
-puts step.info.to_json
+step = Step.load(path.dup)
+
+step.info.each do |k,v|
+  case v
+  when nil
+    next
+  when Exception
+    puts Log.color(:title, "Exception")
+    Log.exception v
+  else
+    puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
+  end
+rescue
+  next
+end