scout-gear 10.8.3 → 10.9.0

data/lib/scout/workflow/usage.rb

@@ -35,6 +35,7 @@ module Task
       str.puts SOPT.input_array_doc(inputs)
 
       inputs.select{|name,type, _| type == :select }.each do |name,_,_,_,options|
+        next unless options
         selects << [name, options[:select_options]] if options[:select_options]
       end
     end
@@ -194,9 +195,9 @@ module Workflow
       if child
         #·description << "->" << task_name.to_s
       elsif first
-        description << "" << task_name.to_s
+        description += "" + task_name.to_s
       else
-        description << ";" << task_name.to_s
+        description += ";" + task_name.to_s
       end
       
       seen << [workflow, task_name]

data/lib/scout/workflow/util.rb

@@ -16,5 +16,27 @@ module Workflow
     name = name.to_sym
     dependencies.select{|dep| dep.task_name.to_sym == name }
   end
+
+  def all_tasks
+    tasks.keys
+  end
+
+  def self.list
+    Path.setup('workflows').glob('*').collect{|p| p.basename }
+  end
+
+  def task_jobs_files(task_name)
+    self.directory[task_name].glob("**").
+      collect{|f| %w(info files).include?(f.get_extension) ? f.unset_extension : f }.
+      uniq
+  end
+
+  def task_jobs(task_name)
+    task_jobs_files(task_name).collect{|f| Step.load f }
+  end
+
+  def load_job(task_name, name)
+    Step.new self.directory[task_name][name]
+  end
 end
 

data/scout-gear.gemspec

@@ -2,21 +2,22 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.8.3 ruby lib
+# stub: scout-gear 10.9.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.8.3".freeze
+  s.version = "10.9.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-06-13"
+  s.date = "1980-01-02"
   s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
     "LICENSE.txt",
+    "README.md",
     "README.rdoc"
   ]
   s.files = [
@@ -24,12 +25,19 @@ Gem::Specification.new do |s|
     ".gitmodules",
     ".vimproject",
     "LICENSE.txt",
+    "README.md",
     "README.rdoc",
     "Rakefile",
     "VERSION",
     "bin/scout",
-    "doc/lib/scout/path.md",
-    "doc/lib/scout/workflow/task.md",
+    "doc/Association.md",
+    "doc/Entity.md",
+    "doc/KnowledgeBase.md",
+    "doc/Persist.md",
+    "doc/Semaphore.md",
+    "doc/TSV.md",
+    "doc/WorkQueue.md",
+    "doc/Workflow.md",
     "lib/scout-gear.rb",
     "lib/scout.rb",
     "lib/scout/association.rb",
@@ -51,6 +59,7 @@ Gem::Specification.new do |s|
     "lib/scout/knowledge_base/query.rb",
     "lib/scout/knowledge_base/registry.rb",
     "lib/scout/knowledge_base/traverse.rb",
+    "lib/scout/monitor.rb",
     "lib/scout/persist/engine.rb",
     "lib/scout/persist/engine/fix_width_table.rb",
     "lib/scout/persist/engine/packed_index.rb",
@@ -129,7 +138,9 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/cat",
     "scout_commands/doc",
+    "scout_commands/entity",
     "scout_commands/find",
     "scout_commands/glob",
     "scout_commands/kb/config",
@@ -143,6 +154,8 @@ Gem::Specification.new do |s|
     "scout_commands/rbbt",
     "scout_commands/resource/produce",
     "scout_commands/resource/sync",
+    "scout_commands/system/clean",
+    "scout_commands/system/status",
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
@@ -250,7 +263,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.6".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4
@@ -259,6 +272,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<matrix>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<csv>.freeze, [">= 0".freeze])
   s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0".freeze])
 end
 

data/scout_commands/cat

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+require 'scout-gear'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Cat a file
+
+$ #{$0} [<options>] (<resource> <path>|<path>)
+
+-h--help Print this help
+-r--requires* Files to require; 'all' for all in Scout.etc.requires
+-lw--load_workflow* Files to require; 'all' for all in Scout.etc.requires
+-w--where* Where to look for the path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+resource, path = ARGV
+path, resource = resource, nil if path.nil?
+
+raise MissingParameterException.new(:path) if path.nil?
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+location = nil
+where = options[:where]
+all = options[:all]
+
+begin
+  resource = Workflow.require_workflow resource
+rescue
+  begin
+    resource = Kernel.const_get(resource)
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end if resource
+
+path = (resource || Scout)[path.dup]
+
+if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
+  location = path.find(where)
+
+  if Array === location
+    file = location.first
+    puts location.first.read
+    puts location * "\n"
+  else
+    file = Open.exists?(location)? location : nil
+  end
+
+  puts file.read if file
+else
+  raise ParameterException, "Where '#{where}' not identified. Try scout-camp if looking for a remote file"
+end

data/scout_commands/doc

@@ -26,7 +26,9 @@ end
 
 module_name = ARGV.first
 if module_name.nil?
-  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| 
+    f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
+  } * "\n"
 else
   begin
     puts Scout.doc.lib.scout[module_name].find_with_extension('md', :lib).read

data/scout_commands/entity

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Execute an entity property
+
+$ #{$0} [<options>] <entity_type> <property> <entity> [<entity_options>] [<property_parameters>]*
+
+Property options are an array of strings that get passed to the property, the
+remaining ARGV elements; property parameters may be optional
+
+-h--help Print this help
+-W--workflows* Workflows to load
+EOF
+
+type, property, entity = ARGV
+
+if options[:help] and type.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise MissingParameterException, :type if type.nil?
+
+workflows = IndiferentHash.process_options options, :workflows, workflows: 'local'
+
+workflows = workflows.split(",")
+
+workflows.each do |workflow|
+  workflow = workflow.strip
+  wf = case workflow
+       when 'local'
+         [:lib, :current].each do |location|
+           Workflow.require_workflow Scout.root['workflow.rb'].find(location) if Scout.root['workflow.rb'].find(location).exists?
+         end
+       when 'lib'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:lib) if Scout.root['workflow.rb'].find(:lib).exists?
+       when 'current'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:current) if Scout.root['workflow.rb'].find(:current).exists?
+       else
+         Workflow.require_workflow workflow
+       end
+
+  begin
+    require "#{wf.to_s}/entity/#{Misc.snake_case(type)}"
+  rescue Exception
+  end
+
+end
+
+begin
+  require "entity/#{Misc.snake_case(type)}"
+rescue Exception
+end
+entity_module = Kernel.const_get type
+
+entity_module.annotations.uniq.each do |attr|
+  SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute" 
+end
+
+if options[:help] and property.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  properties = entity_module.properties
+
+  puts Log.color :title, 'Properties: '
+  puts properties.keys * "\n"
+
+  exit 0
+end
+
+SOPT.synopsys[' [<entity_options>]'] = ""
+
+raise MissingParameterException, :property if property.nil?
+
+
+raise ParameterException, "Property no available" unless entity_module.instance_methods.include? property.to_sym
+
+if options[:help]
+  properties = entity_module.properties[property.to_sym]
+
+  property_str = properties.collect do |type, name|
+    if type != :opt
+      "[<#{name} (#{type})>]"
+    else
+      "[<#{name}>]"
+    end
+  end * " "
+
+  SOPT.synopsys['[<property_parameters>]*'] = property_str
+  
+  if defined? scout_usage
+    scout_usage
+  else
+    puts SOPT.doc
+  end
+
+  exit 0
+end
+
+raise MissingParameterException, :entity if entity.nil?
+
+entity = Entity.prepare_entity(entity, type, options)
+
+entity_params = ARGV[3..-1]
+
+entity_params = entity_params.collect do |param|
+  if Path.located?(param) && Path.can_read?(param)
+    Open.read(param)
+  else
+    case param
+    when 'true'
+      true
+    when 'false'
+      true
+    when /^-?\d+$/
+      param.to_i
+    when /^-?\d+\.\d+$/
+      param.to_f
+    else
+      param
+    end
+  end
+end
+
+res = entity.send(property, *entity_params)
+
+case res
+when nil
+when Array
+  puts res * "\n"
+when String
+  puts res unless res.empty?
+when Symbol, Numeric
+  puts res.to_s
+when Step
+  puts res.path
+else
+  puts res.to_s
+end

data/scout_commands/system/clean

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/monitor'
+
+$0 = "scout #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean orphaned files
+
+$ rbbt system clean <workflow> <task>
+
+Specify workflow '.' and no task to examine the jobs of the current directory (usefull for web-server cache).
+
+-a--all Apply to all jobs, not only uncompleted
+-o--older* Clean jobs not access in some time
+-f--force Remove locks and files regardless of been active
+-q--quick Quick check
+-d--dirty Clean dirty jobs
+-h--help Print this help
+EOF
+
+scout_usage and exit 0 if options[:help]
+
+workflow, task = ARGV
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+force = options.delete :force
+dirty = options.delete :dirty
+time = options.delete :older
+
+ENV["SCOUT_DEBUG_CLEAN"] = 'false'
+ENV["SCOUT_UPDATE"] = 'true' if dirty
+
+time = Misc.timespan time, 'd' if time
+
+puts Log.color(:magenta, "# System clean")
+
+locks = Scout.lock_info
+if locks.any?
+  puts
+  puts Log.color(:magenta, "Locks:")
+  locks.each do |file,info|
+    if force or (info[:pid] && ! Misc.pid_alive?(info[:pid]))
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+persists = Scout.persist_info
+if persists.any?
+  puts
+  puts Log.color(:magenta, "Persist:")
+  persists.each do |file,info|
+    if force or (info[:pid] and Misc.pid_alive? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+sensiblewrites = Scout.sensiblewrite_info
+if sensiblewrites.any?
+  puts
+  puts Log.color(:magenta, "Writes:")
+  sensiblewrites.each do |file,info|
+    if force or (info[:pid] and Misc.pid_alive? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
+puts
+puts Log.color(:magenta, "# Workflow clean")
+puts
+
+if workflow === ['.']
+  jobs = Scout.job_info ["all"], task, ['.']
+else
+  jobs = Scout.job_info workflow, task
+end
+
+workflows = {}
+
+TSV.traverse jobs do |file,i|
+  if options[:quick] and i[:done]
+    status = 'done'
+  else
+    info = begin
+             Open.open(i[:info_file]) do |f|
+               Step.load_info(f)
+             end
+           rescue
+             {:status => :noinfo}
+           end
+
+    done = Open.exists?(file) && ! Open.broken_link?(file)
+    pid = info[:pid] || (Open.exists?(file + '.pid') && Open.read(file + '.pid')) unless done
+
+    status = info[:status].to_s
+    if status != "noinfo"
+      status = :missing if status == "done" and not done
+      status = :nopid if status != "done" and pid.nil?
+      status = :dead if status != "done" and pid and not Misc.pid_alive?(pid)
+      status = :sync if status != "done" and done
+    end
+
+    status = :dirty if info[:status].to_s == "done" and dirty and Workflow.load_step(file).dirty?
+    if info[:status] == :error
+      begin
+        exception = info[:exception][:class]
+        Kernel.const_get exception
+        status = :non_recoverable if exception.superclass === ScoutException
+      rescue
+      end
+    end
+
+    status = status.to_s
+  end
+
+  if time and Open.exists?(file)
+    old = Time.now - Open.atime(file)
+    if old > time
+      status = 'old'
+    end
+  end
+
+  if (force && status !~ /done/) or 
+      status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
+      (status == "noinfo" and not done) or
+      status == "" 
+
+    puts "  Removing #{ file } - #{status}" 
+    Step.clean(file) 
+  end
+end
+