scout-gear 8.1.0 → 9.1.0

data/lib/scout/workflow.rb

@@ -25,20 +25,33 @@ module Workflow
     base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
   end
 
-  def self.require_workflow(workflow_name)
-    workflow = workflow_name 
-    workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
-    workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-    workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
-    if Open.exists?(workflow)
+  def self.require_workflow(workflow_name_orig)
+    first = nil
+    workflow_name_orig.split("+").each do |complete_workflow_name|
       self.main = nil
-      workflow = workflow.find if Path === workflow
-      $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
-      load workflow
-    else
-      raise "Workflow #{workflow_name} not found"
+      workflow_name, *subworkflows = complete_workflow_name.split("::")
+      workflow = workflow_name
+      workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
+      workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
+      workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
+      if Open.exists?(workflow)
+        self.main = nil
+        workflow = workflow.find if Path === workflow
+        $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
+        load workflow
+      else
+        raise "Workflow #{workflow_name} not found"
+      end
+
+      current = begin
+                  Kernel.const_get(complete_workflow_name)
+                rescue
+                  self.main || workflows.last
+                end
+
+      first ||= current
     end
-    self.main || workflows.last
+    first
   end
 
   def job(name, *args)

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 8.1.0 ruby lib
+# stub: scout-gear 9.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "8.1.0"
+  s.version = "9.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-06-15"
+  s.date = "2023-06-30"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
     "lib/scout/persist/path.rb",
     "lib/scout/persist/serialize.rb",
     "lib/scout/resource.rb",
+    "lib/scout/resource/open.rb",
     "lib/scout/resource/path.rb",
     "lib/scout/resource/produce.rb",
     "lib/scout/resource/produce/rake.rb",
@@ -126,6 +127,7 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/step/dependencies.rb",
     "lib/scout/workflow/step/file.rb",
     "lib/scout/workflow/step/info.rb",
+    "lib/scout/workflow/step/inputs.rb",
     "lib/scout/workflow/step/load.rb",
     "lib/scout/workflow/step/progress.rb",
     "lib/scout/workflow/step/provenance.rb",
@@ -143,6 +145,8 @@ Gem::Specification.new do |s|
     "scout_commands/glob",
     "scout_commands/offsite",
     "scout_commands/rbbt",
+    "scout_commands/resource/produce",
+    "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
@@ -151,6 +155,8 @@ Gem::Specification.new do |s|
     "share/color/color_names",
     "share/color/diverging_colors.hex",
     "share/software/install_helpers",
+    "share/templates/command",
+    "share/templates/workflow.rb",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
     "test/scout/log/test_color.rb",

data/scout_commands/resource/produce

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Produce a resource
+
+$ #{$0} [<options>] <Resource> <file> 
+
+-h--help Print this help
+-W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
+-r--requires* Files to require; 'all' for all in Scout.etc.requires:
+-f--force Force the production if the file is already present
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+resource, path = ARGV
+
+begin
+  resource = Kernel.const_get(resource)
+rescue
+  begin
+    resource = Workflow.require_workflow resource
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end
+
+force = options[:force]
+
+puts resource[path].produce(force).find
+

data/scout_commands/template

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Print a template
+
+$ #{$0} [<options>] <template> [<VAR>=<value>]*
+
+Print a template making the substitutions that follow
+
+-h--help Print this help
+-s--sub* List a substitutions as a parameter instead
+-s--var* List a substitutions as a parameter, variable name
+-s--value* List a substitutions as a parameter, value
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+template, *subs = ARGV
+
+raise MissingParameterException, :template if template.nil?
+
+txt = Scout.share.templates[template].find_with_extension('rb').read
+
+if options[:sub]
+  subs ||= []
+  subs << options[:sub]
+end
+
+if options[:var]
+  subs ||= []
+  subs << [options[:var], options[:value]] * "="
+end
+
+
+subs.each do |sub|
+  key, value = sub.split("=")
+
+  txt.gsub!(key, value)
+end
+
+puts txt

data/scout_commands/workflow/install

@@ -49,6 +49,7 @@ Misc.in_dir(workflow_dir) do
       Log.info "Updating: " + workflow
       Misc.in_dir(Misc.snake_case(workflow)) do
         `git pull`
+        `git checkout scout 2> /dev/null`
         `git submodule init`
         `git submodule update`
       end
@@ -56,6 +57,7 @@ Misc.in_dir(workflow_dir) do
       Misc.in_dir(workflow) do
         Log.info "Updating: " + workflow
         `git pull`
+        `git checkout scout 2> /dev/null`
         `git submodule init`
         `git submodule update`
       end
@@ -93,6 +95,7 @@ Misc.in_dir(workflow_dir) do
       end
       Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
       Misc.in_dir(Misc.snake_case(workflow)) do
+        `git checkout scout 2> /dev/null`
         `git submodule init`
         `git submodule update`
       end

data/scout_commands/workflow/task

@@ -18,14 +18,15 @@ $ #{$0} [<options>] <workflow> <task>
 -h--help Print this help
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
--jn--job_name* Name to use as job identifier
+--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
+-jn--jobname* Name to use as job identifier
 -li--load_inputs* Directory with inputs files to load
 -pf--print_filepath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
 -ct--clean_task* Clean a particular task
--d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
+-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 EOF
 
 workflow_name, task_name = ARGV
@@ -38,8 +39,8 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :job_name, :print_filepath, :deploy,
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
   :deploy => 'serial'
 
 if help
@@ -59,6 +60,13 @@ else
   job_inputs = task.get_SOPT
 end
 
+if override_deps
+  override_deps.split($array_separator || ",").each do |part|
+    t_, value = part.split("=")
+    job_inputs.merge!( t_ => value)
+  end
+end
+
 job = workflow.job(task_name, jobname, job_inputs)
 
 job.recursive_clean if recursive_clean
@@ -78,7 +86,7 @@ if clean_task
       dep.set_info :status, :cleaned
     end
 
-    job.clean if job.task_name.to_s == clean_task.to_s
+    job.clean if (job.task_name.to_s == clean_task.to_s) || ! job.updated?
   end
 end
 
@@ -92,9 +100,19 @@ else
     orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
     orchestrator.process({}, job)
   when "slurm"
-    SLURM.orchestrate(job)
+    require 'rbbt-scout'
+    require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
+    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    job.grace
   else 
-    OffsiteStep.setup(job, server: deploy, provided_inputs: job_inputs)
+    if deploy.end_with?('-slurm')
+      server = deploy.sub('-slurm','')
+      OffsiteStep.setup(job, server: server, slurm: true)
+    else
+      OffsiteStep.setup(job, server: deploy)
+    end
+
     job.run
   end unless job.done?
 

data/share/software/install_helpers

@@ -32,6 +32,7 @@ else
 		local basedir="$(realpath $1)"
 		local path="$(realpath "$2")"
     local pre=""
+    basedir=${basedir%.}
     while [[ ! $path = $basedir* ]]; do
       pre=${pre}../
       basedir=$(dirname $basedir)
@@ -44,9 +45,8 @@ function link(){
   local source="$1"
   local target="$2"
   local rel_source=$(relative_path "$(dirname $target)" "$source")
-  
+
   [ -h "$target" ] && rm "$target"
-  echo ln -s "$rel_source" "$target" 1>&2
   [ -h "$target" ] || ln -s "$rel_source" "$target"
 }
 

data/share/templates/command

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+

data/share/templates/workflow.rb

@@ -0,0 +1,14 @@
+Misc.add_libdir if __FILE__ == $0
+
+#require 'scout/sources/MODULE'
+
+module MODULE
+  extend Workflow
+
+
+end
+
+#require 'MODULE/tasks/basic.rb'
+
+#require 'scout/knowledge_base/MODULE'
+#require 'scout/entity/MODULE'

data/test/scout/offsite/test_step.rb

@@ -19,7 +19,6 @@ TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
 
       job = wf.job(:string)
 
-      sss 0
       off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
 
       refute off.done?

data/test/scout/resource/test_produce.rb

@@ -74,4 +74,19 @@ end
       assert_equal list, file.list
     end
   end
+
+  def test_produce_compressed
+    dir = tmpdir.directory[__method__]
+    dir.pkgdir = Scout
+    list = %w(a b)
+
+    Scout.claim dir["foo.gz"], :proc do |filename|
+      list
+    end
+
+    Misc.in_dir(dir) do
+      file = dir.foo
+      assert_include file.produce.find, '.gz'
+    end
+  end
 end

data/test/scout/resource/test_util.rb

@@ -7,6 +7,15 @@ class TestResourceUtil < Test::Unit::TestCase
     assert_equal 'share/data/somedir/somepath', Scout.identify(path)
   end
 
+  def test_identify_with_subdir
+    m = Module.new
+    m.extend Resource
+    m.subdir = 'test/subdir'
+    path = m.root['share/data/somedir/somepath'].find
+    assert_equal 'share/data/somedir/somepath', m.identify(path)
+  end
+
+
   def test_relocate
     TmpFile.with_file do |dir|
       Path.setup dir

data/test/scout/test_persist.rb

@@ -138,8 +138,8 @@ class TestPersist < Test::Unit::TestCase
             Open.write(output2, io)
           end
         end
-        Process.wait pid1
-        Process.wait pid2
+        Process.waitpid pid1
+        Process.waitpid pid2
 
         assert File.exist?(output1) || File.exist?(output2)
         [pid1, pid2].zip([output2, output1]).each do |pid, found|

data/test/scout/test_work_queue.rb

@@ -56,7 +56,7 @@ class TestWorkQueue < Test::Unit::TestCase
       end
     end
 
-    Process.wait pid
+    Process.waitpid pid
     q.close
 
     q.join

data/test/scout/tsv/test_parser.rb

@@ -95,6 +95,27 @@ k a|A b|B
     assert_equal 'a', tsv['k'][0][0]
   end
 
+  def test_parse_head
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+k1 a|A b|B
+k2 a|A b|B
+k3 a|A b|B
+k4 a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, :head => 2)
+    assert_equal 2, tsv.keys.length
+
+    content.rewind
+    tsv = TSV.parse(content, :head => 3)
+    assert_equal 3, tsv.keys.length
+  end
+
+
   def test_parse_fields
     content =<<-EOF
 #: :sep=" "#:type=:double

data/test/scout/workflow/step/test_info.rb

@@ -41,6 +41,5 @@ class TestStepInfo < Test::Unit::TestCase
       refute Path === step1.info[:inputs][0]
 
     end
-
   end
 end

data/test/scout/workflow/step/test_provenance.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/workflow'
 class TestStepProvenance < Test::Unit::TestCase
-  def test_true
+  def test_prov_report
     tmpfile = tmpdir.test_step
     step1 = Step.new tmpfile.step1, ["12"] do |s|
       s.length
@@ -21,5 +21,28 @@ class TestStepProvenance < Test::Unit::TestCase
     assert_include Step.prov_report(step2), 'step1'
     assert_include Step.prov_report(step2), 'step2'
   end
+
+  def __test_input_dependencies
+    wf = Workflow.annonymous_workflow "TaskInputDepProv" do
+      input :value1, :integer
+      task :number1 => :integer do |i| i end
+
+      input :value2, :integer
+      task :number2 => :integer do |i| i end
+
+      input :number1, :integer
+      input :number2, :integer
+      task :sum => :integer do |i1,i2| i1 + i2 end
+
+      dep :number1, :value1 => 1
+      dep :number2, :value1 => 2
+      dep_task :my_sum, self, :sum, :number1 => :number1, :number2 => :number2
+
+    end
+    job = wf.job(:my_sum)
+
+    ppp Step.prov_report(job)
+
+  end
 end
 

data/test/scout/workflow/task/test_dependencies.rb

@@ -20,6 +20,8 @@ class TestTaskDependencies < Test::Unit::TestCase
 
     assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
 
+    assert_equal [step1_job.path], wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).overriden_deps.collect{|d| d.path }
+
     assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).exec
   end
 

data/test/scout/workflow/task/test_inputs.rb

@@ -111,7 +111,6 @@ class TestTaskInput < Test::Unit::TestCase
   def test_save_and_load
     task = self.example_task
 
-    sss 0
     TmpFile.with_file("2\n3") do |integer_array_file|
       inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
       original_digest =  task.process_inputs(inputs).last

data/test/scout/workflow/test_definition.rb

@@ -1,7 +1,7 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-class TestClass < Test::Unit::TestCase
+class TestWorkflowDefinition < Test::Unit::TestCase
   def test_function_task
     wf = Workflow.annonymous_workflow do
       self.name = "StringLength"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 8.1.0
+  version: 9.1.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2023-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
@@ -203,6 +203,7 @@ files:
 - lib/scout/persist/path.rb
 - lib/scout/persist/serialize.rb
 - lib/scout/resource.rb
+- lib/scout/resource/open.rb
 - lib/scout/resource/path.rb
 - lib/scout/resource/produce.rb
 - lib/scout/resource/produce/rake.rb
@@ -252,6 +253,7 @@ files:
 - lib/scout/workflow/step/dependencies.rb
 - lib/scout/workflow/step/file.rb
 - lib/scout/workflow/step/info.rb
+- lib/scout/workflow/step/inputs.rb
 - lib/scout/workflow/step/load.rb
 - lib/scout/workflow/step/progress.rb
 - lib/scout/workflow/step/provenance.rb
@@ -269,6 +271,8 @@ files:
 - scout_commands/glob
 - scout_commands/offsite
 - scout_commands/rbbt
+- scout_commands/resource/produce
+- scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/info
 - scout_commands/workflow/install
@@ -277,6 +281,8 @@ files:
 - share/color/color_names
 - share/color/diverging_colors.hex
 - share/software/install_helpers
+- share/templates/command
+- share/templates/workflow.rb
 - test/scout/indiferent_hash/test_case_insensitive.rb
 - test/scout/indiferent_hash/test_options.rb
 - test/scout/log/test_color.rb