scout-gear 10.7.4 → 10.7.6

data/scout_commands/workflow/cmd

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Run a workflow command
+
+$ #{$0} <workflow> <command> [<subcommands>] [<options>] [<arg> ...]
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+workflow = ARGV.shift
+
+if workflow == '-h'
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise ParameterException, "No workflow specified" if workflow.nil?
+
+require 'scout/workflow'
+
+wf = Workflow.require_workflow workflow
+dir = $command_dir = wf.libdir.share.scout_commands.find
+
+def prev_dir(prev)
+    scout_command_dir = $command_dir
+
+    prev.each do |previous_command|
+        scout_command_dir = scout_command_dir[previous_command]
+    end
+
+    scout_command_dir
+end
+
+def commands(prev)
+    scout_command_dir = prev_dir(prev)
+
+    command_file_dirs = scout_command_dir.find_all
+    command_files = command_file_dirs.collect{|d| d.glob('*') }.flatten
+    command_files.collect{|p| File.basename(p) }.uniq.reject{|p| p =~ /\.desc$/}.sort
+end
+
+
+prev = []
+
+$previous_commands << ' cmd'
+$previous_commands << ' '  << workflow
+
+begin
+  while ARGV.any?
+    command = ARGV.shift
+    case
+    when File.directory?(dir[command].find)
+      prev << command
+      $previous_commands << command
+      dir = dir[command]
+    when File.directory?(dir[command].find)
+      prev << command
+      dir = dir[command]
+    when dir[command].exists?
+      load dir[command].find
+      exit 0
+    else
+      if command == 'bootstrap'
+        if wf.libdir["test_workflow.rb"].exists?
+          Log.info "No bootstrap for #{ workflow }, running test_workflow.rb instead"
+          CMD.cmd_log('ruby', wf.libdir["test_workflow.rb"].find)
+        else
+          Log.info "No bootstrap for #{ workflow }, running examples instead"
+          CMD.cmd_log("scout workflow example #{ workflow }")
+          exit 0
+        end
+      end
+
+      raise ParameterException, "Error: Command not understood: #{command}"
+    end
+  end
+end
+
+puts SOPT.doc
+puts
+puts Log.color :magenta, "## COMMANDS"
+puts
+puts Log.color :magenta, "Command:"
+puts 
+puts " scout #{$previous_commands * " "} "
+puts 
+puts Log.color :magenta, "Subcommands:"
+puts 
+
+commands(prev).each do |command|
+  directory = File.directory? dir[command].find
+  if directory
+    puts "  " << Log.color(:blue, command)
+  else
+    puts "  " << command
+  end
+end
+    

data/scout_commands/workflow/process

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/workflow/deployment/queue'
+require 'scout/workflow/deployment/orchestrator'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Process the queue
+
+$ #{$0} [<options>] ([<workflow>] [<task>] [<name>] | <filename>)
+
+-h--help Print this help
+-l--list List queue jobs
+--continuous Process continuously
+--produce_timer* Produce timer for sleeping
+--produce_cpus* Cpus to use concurrently producing jobs
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+list, continuous = IndiferentHash.process_options options, :list, :continuous
+
+queue_dir = Scout.var.queue
+
+class TrayAgain < Exception; end
+
+begin
+  if ARGV.empty?
+    files = queue_dir.glob("*/*/*")
+    if list
+      puts files * "\n"
+      exit
+    end
+  else
+    workflow, task, name = ARGV
+
+    # First deal with fixed files
+    if task.nil?
+      if Open.exists?(workflow)
+        Workflow.unqueue files.first
+        exit
+
+      else
+        files = queue_dir.glob("#{workflow}/*/*")
+      end
+    elsif name.nil?
+      files = queue_dir.glob("#{workflow}/#{task}/*")
+    else
+      files = queue_dir.glob("#{workflow}/#{task}/#{name}*")
+    end
+  end
+
+  jobs = files.collect{|file| Workflow.queue_job(file) }
+  begin
+    options.keys.each do |key|
+      options[key.to_sym] = options.delete(key)
+    end
+    Workflow.produce(jobs, **options)
+  rescue Workflow::Orchestrator::NoWork
+  end
+
+  files.each do |f| Open.rm_rf f end
+
+  Workflow.job_cache.clear
+
+  if continuous
+    Log.debug "Continuous processing"
+    sleep 1
+    raise Workflow::Orchestrator::NoWork
+  end
+rescue Workflow::Orchestrator::NoWork
+  retry
+end
+

data/scout_commands/workflow/task

@@ -20,8 +20,9 @@ $ #{$0} [<options>] <workflow> <task>
 --update Update jobs with newer dependencies
 --deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 --fork Fork and return path
+--load_inputs* Directory or file with inputs files to load
+--save_inputs* Directory or tar.gz file path to store inputs
 -jn--jobname* Name to use as job identifier
--li--load_inputs* Directory with inputs files to load
 -pf--printpath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
@@ -40,8 +41,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
+help, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
   :deploy => 'serial'
 
 if help
@@ -93,12 +94,23 @@ end
 
 if provenance
   puts Step.prov_report(job)
+
 elsif do_fork
   job.fork
   puts job.path
   exit 0
+elsif save_inputs
+  puts job.save_inputs(save_inputs)
+  exit 0
 else
+
   case deploy
+  when "queue"
+    if ! job.done?
+      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.clean_name].find
+      puts job.save_inputs(save_inputs)
+      exit
+    end
   when "serial"
     job.run(true) 
   when "local"

data/test/data/person/README.md

@@ -0,0 +1,17 @@
+Databases describing people and their relationships.
+
+# Brothers
+
+Sibling relationships. The first of the pair is the Older 
+
+# Marriages
+
+Marriages between people. 
+
+Date: Date when the marriage occurred
+
+# Parents
+
+List parents.
+
+Type of parent: father or mother

data/test/scout/knowledge_base/test_description.rb

@@ -0,0 +1,59 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/knowledge_base'
+class TestKnowledgebaseDesc < Test::Unit::TestCase
+  def test_brothers_registry
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers, description: "Sibling relationships."
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_brothers_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      kb.dir['brothers.md'].write "Sibling relationships."
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_broters_kb_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      kb.dir['README.md'].write <<-EOF
+Databases describing people and their relationships.
+
+# Brothers
+
+Sibling relationships.
+      EOF
+      assert_equal "Sibling relationships.", kb.description(:brothers)
+    end
+  end
+
+  def test_brothers_kb_source_README
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      assert_include kb.description(:brothers), "Sibling relationships."
+    end
+  end
+ 
+  def test_full_description
+    TmpFile.with_dir do |dir|
+      brothers = datafile_test(:person).brothers
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, brothers
+      assert_include kb.markdown(:brothers), "Older"
+    end
+  end
+end
+

data/test/scout/workflow/task/test_dependencies.rb

@@ -120,7 +120,7 @@ class TestTaskDependencies < Test::Unit::TestCase
     job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
     assert_equal 5, job.run
 
-    TmpFile.with_file(4) do |file|
+    TmpFile.with_file("4") do |file|
       job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
       assert_equal 6, job.run
       assert_not_equal Task::DEFAULT_NAME, job.name
@@ -238,7 +238,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1  do |id,options|
+      dep :step1, :input1 => "1"  do |id,options|
         {:inputs => options}
       end
       task :step2 => :string do |i| step(:step1).load end
@@ -271,7 +271,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1  do |id,options|
+      dep :step1, :input1 => "1"  do |id,options|
         [{:inputs => options}]
       end
       input :input2, :string
@@ -289,7 +289,7 @@ class TestTaskDependencies < Test::Unit::TestCase
       input :input1, :string
       task :step1 => :string do |i| i end
 
-      dep :step1, :input1 => 1
+      dep :step1, :input1 => "1"
       input :input2, :string
       task :step2 => :string do |i| step(:step1).load end
     end
@@ -304,9 +304,9 @@ class TestTaskDependencies < Test::Unit::TestCase
     wf = Workflow.annonymous_workflow "TaskInputs" do
       input :input1, :integer, "", 1
       input :input2, :integer, "", 0
-      task :step1 => :string do |i1,i2| i1 + i2 end
+      task :step1 => :string do |i1,i2| (i1 + i2).to_s end
 
-      dep :step1, :input2 => 1
+      dep :step1, :input2 => "1"
       task :step2 => :string do |i| step(:step1).load end
 
       dep :step2
@@ -334,7 +334,7 @@ class TestTaskDependencies < Test::Unit::TestCase
         if i1 < 0
           raise ScoutException
         else
-          i1
+          i1.to_s
         end
       end
 

data/test/scout/workflow/test_definition.rb

@@ -64,7 +64,7 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     refute Open.exist?(dep_path)
     Scout::Config::CACHE.replace old_cache
     assert_include job.archived_info, dep_path
-    assert_equal :done, job.archived_info[dep_path][:status]
+    assert_equal :done, job.archived_info[dep_path][:status].to_sym
   end
 
   def test_task_alias_remove_dep_partial
@@ -95,7 +95,7 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     assert call_name.done?
     Scout::Config::CACHE.replace old_cache
     assert_include job.archived_info, call_name.path
-    assert_equal :done, job.archived_info[call_name.path][:status]
+    assert_equal :done, job.archived_info[call_name.path][:status].to_sym
   end
 end
 

data/test/scout/workflow/test_entity.rb

@@ -0,0 +1,58 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflowEntity < Test::Unit::TestCase
+  def get_EWF
+    @m ||= Module.new do
+      extend EntityWorkflow
+
+      self.name = 'TestEWF'
+
+      property :introduction do
+        "Mi name is #{self}"
+      end
+
+      entity_task hi: :string do
+        "Hi. #{entity.introduction}"
+      end
+
+      list_task group_hi: :string do
+        "Here is the group: " + entity_list.hi * "; "
+      end
+
+      list_task bye: :array do
+        entity_list.collect do |e|
+          "Bye from #{e}"
+        end
+      end
+    end
+  end
+
+  def test_property_job
+    ewf = get_EWF
+
+    e = ewf.setup("Miki")
+
+    assert_equal "Mi name is Miki", e.introduction
+    assert_equal "Hi. Mi name is Miki", e.hi
+  end
+
+  def test_list
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.hi.length
+
+    assert_include l.group_hi, "group: "
+  end
+
+  def test_multiple
+    ewf = get_EWF
+    
+    l = ewf.setup(["Miki", "Clei"])
+
+    assert_equal 2, l.bye.length
+  end
+end
+

data/test/scout/workflow/test_step.rb

@@ -387,7 +387,7 @@ class TestWorkflowStep < Test::Unit::TestCase
       step1.fork(false, sem)
       step2.fork(false, sem)
       sleep 1
-      assert((step1.status == :queue) || (step2.status == :queue))
+      assert((step1.status.to_sym == :queue) || (step2.status.to_sym == :queue))
       step1.join
       step2.join
       assert_equal "done2", step2.run

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.4
+  version: 10.7.6
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-03-10 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -65,34 +65,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.12'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: juwelier
   requirement: !ruby/object:Gem::Requirement
@@ -107,22 +79,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.0
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Temporary files, logs, path, resources, persistence, workflows, TSV,
-  etc.
+description: 'Scout gear: workflow, TSVs, persistence, entities, associations, and
+  knowledge_bases.'
 email: mikisvaz@gmail.com
 executables:
 - scout
@@ -156,17 +114,13 @@ files:
 - lib/scout/entity/object.rb
 - lib/scout/entity/property.rb
 - lib/scout/knowledge_base.rb
+- lib/scout/knowledge_base/description.rb
 - lib/scout/knowledge_base/enrichment.rb
 - lib/scout/knowledge_base/entity.rb
 - lib/scout/knowledge_base/list.rb
 - lib/scout/knowledge_base/query.rb
 - lib/scout/knowledge_base/registry.rb
 - lib/scout/knowledge_base/traverse.rb
-- lib/scout/offsite.rb
-- lib/scout/offsite/exceptions.rb
-- lib/scout/offsite/ssh.rb
-- lib/scout/offsite/step.rb
-- lib/scout/offsite/sync.rb
 - lib/scout/persist/engine.rb
 - lib/scout/persist/engine/fix_width_table.rb
 - lib/scout/persist/engine/packed_index.rb
@@ -214,10 +168,14 @@ files:
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/deployment.rb
 - lib/scout/workflow/deployment/orchestrator.rb
+- lib/scout/workflow/deployment/queue.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
+- lib/scout/workflow/entity.rb
 - lib/scout/workflow/exceptions.rb
+- lib/scout/workflow/export.rb
 - lib/scout/workflow/path.rb
+- lib/scout/workflow/persist.rb
 - lib/scout/workflow/step.rb
 - lib/scout/workflow/step/archive.rb
 - lib/scout/workflow/step/children.rb
@@ -252,14 +210,15 @@ files:
 - scout_commands/kb/show
 - scout_commands/kb/traverse
 - scout_commands/log
-- scout_commands/offsite
 - scout_commands/rbbt
 - scout_commands/resource/produce
 - scout_commands/template
 - scout_commands/update
+- scout_commands/workflow/cmd
 - scout_commands/workflow/info
 - scout_commands/workflow/install
 - scout_commands/workflow/list
+- scout_commands/workflow/process
 - scout_commands/workflow/prov
 - scout_commands/workflow/task
 - scout_commands/workflow/trace
@@ -269,6 +228,7 @@ files:
 - share/software/install_helpers
 - share/templates/command
 - share/templates/workflow.rb
+- test/data/person/README.md
 - test/data/person/brothers
 - test/data/person/identifiers
 - test/data/person/marriages
@@ -281,16 +241,13 @@ files:
 - test/scout/entity/test_named_array.rb
 - test/scout/entity/test_object.rb
 - test/scout/entity/test_property.rb
+- test/scout/knowledge_base/test_description.rb
 - test/scout/knowledge_base/test_enrichment.rb
 - test/scout/knowledge_base/test_entity.rb
 - test/scout/knowledge_base/test_list.rb
 - test/scout/knowledge_base/test_query.rb
 - test/scout/knowledge_base/test_registry.rb
 - test/scout/knowledge_base/test_traverse.rb
-- test/scout/offsite/test_ssh.rb
-- test/scout/offsite/test_step.rb
-- test/scout/offsite/test_sync.rb
-- test/scout/offsite/test_task.rb
 - test/scout/persist/engine/test_fix_width_table.rb
 - test/scout/persist/engine/test_packed_index.rb
 - test/scout/persist/engine/test_sharder.rb
@@ -308,7 +265,6 @@ files:
 - test/scout/test_association.rb
 - test/scout/test_entity.rb
 - test/scout/test_knowledge_base.rb
-- test/scout/test_offsite.rb
 - test/scout/test_semaphore.rb
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
@@ -351,6 +307,7 @@ files:
 - test/scout/workflow/task/test_inputs.rb
 - test/scout/workflow/test_definition.rb
 - test/scout/workflow/test_documentation.rb
+- test/scout/workflow/test_entity.rb
 - test/scout/workflow/test_path.rb
 - test/scout/workflow/test_step.rb
 - test/scout/workflow/test_task.rb

data/lib/scout/offsite/exceptions.rb

@@ -1,9 +0,0 @@
-class SSHProcessFailed < StandardError
-  attr_accessor :host, :cmd
-  def initialize(host, cmd)
-    @host = host
-    @cmd = cmd
-    message = "SSH server #{host} failed cmd '#{cmd}'" 
-    super(message)
-  end
-end