scout-gear 2.0.0 → 5.2.0

data/lib/scout/workflow/usage.rb

@@ -0,0 +1,314 @@
+require 'scout/simple_opt'
+
+module Task
+  def usage(workflow = nil, deps = nil)
+    str = StringIO.new
+    str.puts Log.color(:yellow, name)
+    str.puts Log.color(:yellow, "-" * name.length)
+    str.puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
+    str.puts
+
+    selects = []
+    if inputs && inputs.any?
+      str.puts Log.color(:magenta, "Inputs")
+      str.puts
+      str.puts SOPT.input_array_doc(inputs)
+      str.puts
+    end
+
+    if deps and deps.any?
+      seen = inputs.collect{|name,_| name }
+      dep_inputs = {}
+      deps.each do |dep_workflow,task_name|
+        task = dep_workflow.tasks[task_name]
+        next if task.inputs.nil?
+        inputs = task.inputs.reject{|name, _| seen.include? name }
+        next unless inputs.any?
+        dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
+        dep_inputs[dep] = inputs
+      end
+
+      str.puts Log.color(:magenta, "Inputs from dependencies:") if dep_inputs.any?
+      dep_inputs.each do |dep,inputs|
+        str.puts
+        str.puts Log.color :yellow, dep + ":"
+        str.puts
+        str.puts SOPT.input_array_doc(inputs)
+        str.puts
+      end
+
+      #task_inputs = dep_inputs deps, workflow
+      #task_inputs.each do |task,new_inputs|
+      #  new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t| 
+      #    t.to_sym == :select and task.input_options[i][:select_options] 
+      #  end.each do |i,t| 
+      #    selects << [i, task.input_options[i][:select_options]] 
+      #  end
+
+      #  next if new_inputs.empty?
+
+      #  if task.workflow and task.workflow != workflow
+      #    puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
+      #  else
+      #    puts "  #{Log.color :yellow, task.name.to_s}:"
+      #  end
+      #  puts unless Log.compact
+      #  puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
+      #  puts unless Log.compact
+      #end
+      #puts
+    end
+
+    case
+    when inputs && inputs.select{|name,type| type == :array }.any?
+      str.puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
+      also works in addition to the others. You may use the '--array_separator' option
+      the change this default. Whenever a file is specified it may also accept STDIN using
+      the '-' character."))
+      str.puts
+
+    when inputs && inputs.select{|name,type| type == :file || type == :tsv }.any?
+      str.puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
+      str.puts
+    end
+
+    str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
+    str.puts
+
+    if selects.any?
+      str.puts Log.color(:magenta, "Input select options")
+      str.puts
+      selects.collect{|p| p}.uniq.each do |input,options|
+        str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
+        str.puts unless Log.compact
+      end
+      str.puts
+    end
+    str.rewind
+    str.read
+  end
+
+  def SOPT_str
+    sopt_options = []
+    self.recursive_inputs.each do |name,type,desc,default,options|
+      shortcut = (options && options[:shortcut]) || name.to_s.slice(0,1)
+      boolean = type == :boolean
+
+      sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
+    end
+
+    sopt_options * ":"
+  end
+
+  def get_SOPT(task)
+    sopt_option_string = self.SOPT_str
+    SOPT.get sopt_option_string
+  end
+end
+
+module Workflow
+
+  def dep_tree(task_name, seen = nil)
+    @dep_tree ||= {}
+    key = [self, task_name]
+
+    return @dep_tree[key] if @dep_tree.include?(key)
+    save = seen.nil?
+    seen = Set.new if seen.nil?
+
+    dep_tree = {}
+    task = self.tasks[task_name]
+    task.deps.each do |dep|
+      next if seen.include? dep
+      seen << dep
+      workflow, task, *rest = dep
+
+      key = [workflow, task]
+
+      dep_tree[key] = workflow.dep_tree(task, seen)
+    end if task.deps
+
+    @dep_tree[key] = dep_tree if save
+
+    dep_tree
+  end
+
+  def recursive_deps(task_name)
+    dependencies = []
+    dep_tree(task_name, dependencies)
+    dependencies
+  end
+
+  def _prov_tasks(tree)
+    tasks = [] 
+    heap = [tree]
+    while heap.any?
+      t = heap.pop
+      t.each do |k,v|
+        tasks << k
+        heap << v
+      end
+    end
+    tasks
+  end
+
+  def prov_string(tree)
+    description = ""
+
+    last = nil
+    seen = Set.new
+
+    tasks = _prov_tasks(tree)
+    tasks.each do |workflow,task_name|
+
+      next if seen.include?([workflow,task_name])
+
+      child = last && last.include?([workflow, task_name])
+      first = last.nil?
+      last = _prov_tasks(workflow.dep_tree(task_name))
+
+      break if child
+
+      if child
+        description << "->" << task_name.to_s
+      elsif first
+        description << "" << task_name.to_s
+      else
+        description << ";" << task_name.to_s
+      end
+      
+      seen << [workflow, task_name]
+    end
+    description
+  end
+
+  def prov_tree(tree, offset = 0, seen = [])
+
+    return "" if tree.empty?
+
+    lines = []
+
+    offset_str = " " * offset
+
+    lines << offset_str 
+
+    tree.each do |p,dtree| 
+      next if seen.include?(p)
+      seen.push(p)
+      workflow, task = p
+      lines << offset_str + [workflow.to_s, task.to_s] * "#" + "\n" + workflow.prov_tree(dtree, offset + 1, seen)
+    end
+
+    lines * "\n"
+  end
+
+  def usage(task = nil, abridge = false)
+
+    str = StringIO.new
+
+    if self.documentation[:title] and not self.documentation[:title].empty?
+      title = self.name + " - " + self.documentation[:title]
+      str.puts Log.color :magenta, title
+      str.puts Log.color :magenta, "=" * title.length
+    else
+      str.puts Log.color :magenta, self.name 
+      str.puts Log.color :magenta, "=" * self.name.length
+    end
+
+    if self.documentation[:description] and not self.documentation[:description].empty?
+      str.puts
+      str.puts Misc.format_paragraph self.documentation[:description] 
+      str.puts
+    end
+
+
+    if task.nil?
+
+      str.puts Log.color :magenta, "## TASKS"
+      if self.documentation[:task_description] and not self.documentation[:task_description].empty?
+        str.puts
+        str.puts Misc.format_paragraph self.documentation[:task_description] 
+      end
+      str.puts
+
+      final = Set.new
+      not_final = Set.new
+      tasks.each do |name,task|
+        tree = dep_tree(name)
+        not_final += tree.keys
+        final << name unless not_final.include?(name)
+      end
+
+      not_final.each do |p|
+        final -= [p.last]
+      end
+
+      tasks.each do |name,task|
+        description = task.description || ""
+        description = description.split("\n\n").first
+
+        next if abridge && ! final.include?(name)
+        str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
+
+        prov_string = prov_string(dep_tree(name))
+        str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
+      end
+
+    else
+
+      if Task === task
+        task_name = task.name
+      else
+        task_name = task
+        task = self.tasks[task_name]
+      end
+
+      #dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
+      str.puts task.usage(self, self.recursive_deps(task_name))
+
+      dep_tree = {[self, task_name] => dep_tree(task_name)}
+      prov_tree = prov_tree(dep_tree)
+      if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
+
+        str.puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
+        str.puts
+        prov_tree.split("\n").each do |line|
+          next if line.strip.empty?
+          if m = line.match(/^( *)(\w+?)#(\w*)/i)
+              offset, workflow, task_name =  m.values_at 1, 2, 3
+              str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
+          else
+            str.puts Log.color :blue, line 
+          end
+        end
+        str.puts
+      end
+    end
+
+    str.rewind
+    str.read
+  end
+
+  def SOPT_str(task)
+    sopt_options = []
+    self.tasks[task].recursive_inputs.each do |name,type,desc,default,options|
+      shortcut = options[:shortcut] || name.to_s.slice(0,1)
+      boolean = type == :boolean
+
+      sopt_options << "-#{short}--#{name}#{boolean ? "" : "*"}"
+    end
+
+    sopt_options * ":"
+  end
+
+  def get_SOPT(task)
+    sopt_option_string = self.SOPT_str(task)
+    SOPT.get sopt_option_string
+  end
+
+  def self.get_SOPT(workflow, task)
+    workflow = Workflow.require_workflow workflow if String === workflow
+    task = workflow.tasks[task.to_sym] if String === task || Symbol === task
+    workflow.get_SOPT(task)
+  end
+end

data/lib/scout/workflow/util.rb

@@ -0,0 +1,11 @@
+module Workflow
+  def self.annonymous_workflow(name = nil, &block)
+    mod = Module.new
+    mod.extend Workflow
+    mod.name = name
+    mod.directory = Workflow.directory[name] if name
+    mod.instance_eval(&block)
+    mod
+  end
+end
+

data/lib/scout/workflow.rb

@@ -0,0 +1,40 @@
+require_relative 'workflow/definition'
+require_relative 'workflow/util'
+require_relative 'workflow/task'
+require_relative 'workflow/step'
+require_relative 'workflow/documentation'
+require_relative 'workflow/usage'
+
+require_relative 'resource'
+require_relative 'resource/scout'
+
+module Workflow
+  class << self
+    attr_accessor :workflows
+    def workflows
+      @workflows ||= []
+    end
+  end
+
+  attr_accessor :libdir
+  def self.extended(base)
+    self.workflows << base
+    libdir = Path.caller_lib_dir
+    return if libdir.nil?
+    base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
+  end
+
+  def self.require_workflow(workflow)
+    workflow = Path.setup('workflows')[workflow]["workflow.rb"] unless Open.exists?(workflow)
+    if Open.exists?(workflow)
+      workflow = workflow.find if Path === workflow
+      load workflow
+    end
+    workflows.last
+  end
+
+  def job(name, *args)
+    task = tasks[name]
+    task.job(*args)
+  end
+end

data/lib/scout-gear.rb

@@ -1,5 +1,9 @@
+require_relative 'scout/exceptions'
 require_relative 'scout/indiferent_hash'
 require_relative 'scout/tmpfile'
 require_relative 'scout/log'
 require_relative 'scout/path'
 require_relative 'scout/simple_opt'
+require_relative 'scout/resource'
+require_relative 'scout/resource/scout'
+require_relative 'scout/persist'

data/lib/scout.rb

@@ -0,0 +1 @@
+require 'workflow-scout'

data/lib/workflow-scout.rb

@@ -0,0 +1,2 @@
+require 'scout-gear'
+require 'scout/workflow'

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 2.0.0 ruby lib
+# stub: scout-gear 5.2.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "2.0.0"
+  s.version = "5.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-03-30"
+  s.date = "2023-04-28"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -29,6 +29,9 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/scout",
     "lib/scout-gear.rb",
+    "lib/scout.rb",
+    "lib/scout/cmd.rb",
+    "lib/scout/concurrent_stream.rb",
     "lib/scout/exceptions.rb",
     "lib/scout/indiferent_hash.rb",
     "lib/scout/indiferent_hash/case_insensitive.rb",
@@ -42,10 +45,30 @@ Gem::Specification.new do |s|
     "lib/scout/log/progress/util.rb",
     "lib/scout/meta_extension.rb",
     "lib/scout/misc.rb",
+    "lib/scout/misc/digest.rb",
+    "lib/scout/misc/filesystem.rb",
     "lib/scout/misc/format.rb",
+    "lib/scout/misc/insist.rb",
+    "lib/scout/misc/monitor.rb",
+    "lib/scout/open.rb",
+    "lib/scout/open/lock.rb",
+    "lib/scout/open/remote.rb",
+    "lib/scout/open/stream.rb",
+    "lib/scout/open/util.rb",
     "lib/scout/path.rb",
     "lib/scout/path/find.rb",
+    "lib/scout/path/tmpfile.rb",
     "lib/scout/path/util.rb",
+    "lib/scout/persist.rb",
+    "lib/scout/persist/open.rb",
+    "lib/scout/persist/path.rb",
+    "lib/scout/persist/serialize.rb",
+    "lib/scout/resource.rb",
+    "lib/scout/resource/path.rb",
+    "lib/scout/resource/produce.rb",
+    "lib/scout/resource/produce/rake.rb",
+    "lib/scout/resource/scout.rb",
+    "lib/scout/resource/util.rb",
     "lib/scout/simple_opt.rb",
     "lib/scout/simple_opt/accessor.rb",
     "lib/scout/simple_opt/doc.rb",
@@ -53,26 +76,75 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/parse.rb",
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
+    "lib/scout/workflow.rb",
+    "lib/scout/workflow/definition.rb",
+    "lib/scout/workflow/documentation.rb",
+    "lib/scout/workflow/step.rb",
+    "lib/scout/workflow/step/info.rb",
+    "lib/scout/workflow/step/load.rb",
+    "lib/scout/workflow/task.rb",
+    "lib/scout/workflow/task/inputs.rb",
+    "lib/scout/workflow/usage.rb",
+    "lib/scout/workflow/util.rb",
+    "lib/workflow-scout.rb",
     "scout-gear.gemspec",
+    "scout_commands/alias",
+    "scout_commands/find",
+    "scout_commands/glob",
+    "scout_commands/rbbt",
+    "scout_commands/workflow/info",
+    "scout_commands/workflow/list",
+    "scout_commands/workflow/task",
+    "scout_commands/workflow/task_old",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
     "test/scout/log/test_progress.rb",
+    "test/scout/misc/test_digest.rb",
+    "test/scout/misc/test_filesystem.rb",
+    "test/scout/misc/test_insist.rb",
+    "test/scout/open/test_lock.rb",
+    "test/scout/open/test_remote.rb",
+    "test/scout/open/test_stream.rb",
+    "test/scout/open/test_util.rb",
     "test/scout/path/test_find.rb",
     "test/scout/path/test_util.rb",
+    "test/scout/persist/test_open.rb",
+    "test/scout/persist/test_path.rb",
+    "test/scout/persist/test_serialize.rb",
+    "test/scout/resource/test_path.rb",
+    "test/scout/resource/test_produce.rb",
+    "test/scout/resource/test_util.rb",
+    "test/scout/simple_opt/test_doc.rb",
     "test/scout/simple_opt/test_get.rb",
     "test/scout/simple_opt/test_parse.rb",
     "test/scout/simple_opt/test_setup.rb",
+    "test/scout/test_cmd.rb",
+    "test/scout/test_concurrent_stream.rb",
     "test/scout/test_indiferent_hash.rb",
     "test/scout/test_log.rb",
     "test/scout/test_meta_extension.rb",
     "test/scout/test_misc.rb",
+    "test/scout/test_open.rb",
     "test/scout/test_path.rb",
+    "test/scout/test_persist.rb",
+    "test/scout/test_resource.rb",
     "test/scout/test_tmpfile.rb",
-    "test/test_helper.rb"
+    "test/scout/test_workflow.rb",
+    "test/scout/workflow/step/test_info.rb",
+    "test/scout/workflow/step/test_load.rb",
+    "test/scout/workflow/task/test_inputs.rb",
+    "test/scout/workflow/test_definition.rb",
+    "test/scout/workflow/test_documentation.rb",
+    "test/scout/workflow/test_step.rb",
+    "test/scout/workflow/test_task.rb",
+    "test/scout/workflow/test_usage.rb",
+    "test/scout/workflow/test_util.rb",
+    "test/test_helper.rb",
+    "test/test_scout-gear.rb"
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.4.6".freeze
+  s.rubygems_version = "3.4.11".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/alias

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require 'scout-gear'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Creates a one work comman alias.
+
+$ #{$0} <alias> [<options>] [<othercmd>]*
+
+The other commands might include options. But beware the some might reach the alias if they where consumed
+earlier, like --dev or --log. Edit #{Scout.ect.cmd_alias.find} to fine tune.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+cmd, *rest = ARGV
+
+aliases ||= Scout.etc.cmd_alias.exists? ? Scout.etc.cmd_alias.yaml : {}
+
+if cmd.nil?
+  aliases.each do |name, parts|
+    parts = parts.collect{|p|
+      p =~ /^[\w:_\/-]*$/ ? p : "'" << p << "'"
+    }
+    puts [Log.color(:magenta, name), parts * " "] * ": "
+  end
+  exit 0
+end
+
+if rest.empty?
+  aliases.delete cmd
+else
+  aliases[cmd] = rest
+end
+
+Open.write(Scout.etc.cmd_alias.find,(aliases.to_yaml))
+

data/scout_commands/find

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require 'scout-gear'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find files
+
+$ #{$0} [<options>] (<resource> <path>|<path>)
+
+-h--help Print this help
+-r--requires* Files to require; 'all' for all in Scout.etc.requires
+-lw--load_workflow* Files to require; 'all' for all in Scout.etc.requires
+-w--where* Where to look for the path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+resource, path = ARGV
+path, resource = resource, nil if path.nil?
+
+raise MissingParameterException.new(:path) if path.nil?
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+location = nil
+where = options[:where]
+all = options[:all]
+
+begin
+  resource = Workflow.require_workflow resource
+rescue
+  begin
+    resource = Kernel.const_get(resource)
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end if resource
+
+path = (resource || Scout)[path.dup]
+
+if where.nil? || where == 'all' || path.search_paths.include?(where.to_sym)
+  location = path.find(where)
+
+  if Array === location
+    puts location * "\n"
+  else
+    location = nil if ! Open.exists?(location)
+    puts location
+  end
+else
+  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all))
+end

data/scout_commands/rbbt

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+case
+when File.exist?(file = File.join(__dir__, '../modules/rbbt-util/bin/rbbt'))
+  $LOAD_PATH.unshift(lib_dir = File.join(file, '../..', 'lib'))
+  require_relative File.join(lib_dir,'rbbt-util')
+  Path.add_path(:rbbt, File.expand_path(File.join(File.dirname(lib_dir), "/{TOPLEVEL}/{SUBPATH}")))
+  Path.default_pkgdir = Rbbt
+  load file
+when File.exist?(file = File.join(ENV["HOME"], 'git/scout-gear/modules/rbbt-util/bin/rbbt'))
+  $LOAD_PATH.unshift(lib_dir = File.join(file, '../..', 'lib'))
+  require_relative File.join(lib_dir,'rbbt-util')
+  Path.add_path(:rbbt, File.expand_path(File.join(File.dirname(lib_dir), "/{TOPLEVEL}/{SUBPATH}")))
+  Path.default_pkgdir = Rbbt
+  load file
+when File.exist?(file = File.join(ENV["HOME"], 'git/rbbt-util/bin/rbbt'))
+  $LOAD_PATH.unshift(lib_dir = File.join(file, '../..', 'lib'))
+  require_relative File.join(lib_dir,'rbbt-util')
+  Path.add_path(:rbbt, File.expand_path(File.join(File.dirname(lib_dir), "/{TOPLEVEL}/{SUBPATH}")))
+  Path.default_pkgdir = Rbbt
+  load file
+else
+  raise "Cant find rbbt"
+end