scout-gear 9.0.0 → 9.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98b3f84350f2f451b717f004bfc97cd7767f7e0fed58e1a2d0cd023bc700185f
-  data.tar.gz: '00847ad4d92b1cab22305ea867dc12c870b22e192e79536917f82aa90cc82989'
+  metadata.gz: bbe916f2d82030e26b1049319216f45341b7e7f7410df65e3953c0a733af2c21
+  data.tar.gz: 152b31b552f90b2bb06aed9bc3cdd12a3bff48746468e52b39e5754977a40f90
 SHA512:
-  metadata.gz: 4af5e13873fa13dea33161e9200aa725bb4329899e9242ae6504ffae2ab82e7eb01b82a53edc2a4331ede06d0f12bb07f86f579cb29ecbf0d36e791bf6d53f3e
-  data.tar.gz: fc773eb933e5c851aa1d642e872df02d282fbcb29ee1b8e1bf150ffebf9bd4ce8c3670d43a284ff2c8197477a8cf6f71dfcd49b265ad938c723292ec57579998
+  metadata.gz: 51f41961b2a95d847e4a60a70a2774c34098c4fa502818be8b9d5ec30ced53fce81d02862a1104498060c04a88138ca329ae82ba09fa264b6d9ec0daca80a3a4
+  data.tar.gz: 113d51ff9798416073a86debb07ce47708254a4594e5785cfc4543794c3ff4e09da8809c29d67f64d57aeacd30c1c2ad49998d5eca4f544f3381991c296db1c5

data/VERSION

@@ -1 +1 @@
-9.0.0
+9.1.0

data/bin/scout

@@ -99,7 +99,6 @@ if config_keys = options.delete(:config_keys)
   end
 end
 
-
 $scout_command_dir = Scout.bin.scout
 $scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 

data/lib/rbbt-scout.rb

@@ -1,7 +1,8 @@
 $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
-module Rbbt
+module Scout
   extend Resource
   self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end
+Rbbt = Scout
 
 Resource.set_software_env Rbbt.software

data/lib/scout/config.rb

@@ -4,7 +4,7 @@ require_relative 'resource/scout'
 
 module Scout::Config
 
-  CACHE = IndiferentHash.setup({})
+  CACHE ||= IndiferentHash.setup({})
 
   GOT_KEYS=[]
 

data/lib/scout/indiferent_hash/options.rb

@@ -6,7 +6,7 @@ module IndiferentHash
     defaults = string2hash defaults if String === defaults
 
     defaults.each do |key, value|
-      next if options.include? key
+      next if options.include?(key)
 
       options[key] = value 
     end

data/lib/scout/misc/digest.rb

@@ -29,7 +29,7 @@ module Misc
         obj.to_s
       when Float
         if obj % 1 == 0
-          obj.to_i
+          obj.to_i.to_s
         elsif obj.abs > 10
           "%.1f" % obj
         elsif obj.abs > 1

data/lib/scout/path/util.rb

@@ -86,6 +86,26 @@ module Path
     self.annotate(self.split(".")[0..-2] * ".")
   end
 
+  def remove_extension(extension = nil)
+    if extension.nil?
+      unset_extension
+    else
+      self.annotate(self.sub(/\.#{extension}$/,''))
+    end
+  end
+
+  def replace_extension(new_extension = nil, multiple = false)
+    if String === multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
+    elsif multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
+    else
+      new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
+    end
+    new_path = new_path + "." + new_extension.to_s
+    self.annotate(new_path)
+  end
+
 
   # Is 'file' newer than 'path'? return non-true if path is newer than file
   def self.newer?(path, file, by_link = false)

data/lib/scout/resource/util.rb

@@ -28,7 +28,11 @@ module Resource
               m.named_captures.include?(c) ? m[c] : nil
             }.compact * "/"
             unlocated.gsub!(/\/+/,'/')
-            unlocated[self.subdir] = "" if self.subdir
+            if self.subdir && ! self.subdir.empty?
+              subdir = self.subdir
+              subdir += "/" unless subdir.end_with?("/")
+              unlocated[subdir] = "" 
+            end
             choices << self.annotate(unlocated)
           end
         end

data/lib/scout/tsv/dumper.rb

@@ -19,6 +19,7 @@ module TSV
         preamble_str = nil
       end
 
+      preamble_str = preamble_str.strip if preamble_str
       [preamble_str, fields_str].compact * "\n"
     end
 

data/lib/scout/tsv/open.rb

@@ -20,7 +20,7 @@ module Open
         into.add *res
       when TSV, Hash
         key, value = res
-        if into.type == :double
+        if TSV === into && into.type == :double
           into.zip_new key, value, insitu: false
         else
           into[key] = value
@@ -118,6 +118,12 @@ module Open
                 callback.call res if callback
                 nil
               end
+            when Hash
+              obj.each do |key,value|
+                res = block.call(key,value)
+                callback.call res if callback
+                nil
+              end
             when Array
               obj.each do |line|
                 res = block.call(line)

data/lib/scout/tsv/util.rb

@@ -122,7 +122,7 @@ Example:
   end
 
   def digest_str
-    fingerprint
+    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
   end
 
   def inspect

data/lib/scout/workflow/step/dependencies.rb

@@ -7,11 +7,12 @@ class Step
       direct_deps << dep
     end
     seen.concat direct_deps.collect{|d| d.path }
-    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, [])); acc }
+    direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
   end
 
   def recursive_inputs
-    recursive_inputs = @inputs.to_hash
+    recursive_inputs = NamedArray === @inputs ? @inputs.to_hash : {}
+    return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
       acc.merge(dep.recursive_inputs)
     end

data/lib/scout/workflow/step/file.rb

@@ -2,7 +2,11 @@ class Step
   def files_dir
     @files_dir ||= begin
                      dir = @path + ".files"
-                     @path.annotate(dir) if Path === @path
+                     if Path === @path
+                       @path.annotate(dir)
+                     else
+                       Path.setup(dir)
+                     end
                      dir.pkgdir = self
                      dir
                    end

data/lib/scout/workflow/step/info.rb

@@ -9,12 +9,18 @@ class Step
                    end
   end
 
+  def self.load_info(info_file)
+    info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(info)
+  end
+
   def load_info
-    @info = Persist.load(info_file, SERIALIZER) || {}
-    IndiferentHash.setup(@info)
+    @info = Step.load_info(info_file)
     @info_load_time = Time.now
   end
 
+
+
   def save_info(info = nil)
     Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now

data/lib/scout/workflow/step/provenance.rb

@@ -83,10 +83,25 @@ class Step
       end
     end
 
-    str << "\n"
+    str
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
+  def self.prov_indent(step, offset, input_dependencies)
+    return (" " * (offset))
+    if step.alias?
+      (" " * offset + "🡵")
+    elsif step.overriden_task
+      (" " * offset + "🡇")
+    elsif input_dependencies.include?(step)
+      (" " * offset + "┝")
+    elsif step.input_dependencies.any?
+      (" " * offset + "╰")
+    else
+      (" " * (offset+1))
+    end
+  end
+
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil, input_dependencies = nil)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -101,7 +116,7 @@ class Step
 
     this_step_msg = prov_report_msg(status, name, path, info, input)
 
-    input_dependencies = {}
+    input_dependencies ||= {}
     step.dependencies.each do |dep|
       if dep.input_dependencies.any?
         dep.input_dependencies.each do |id|
@@ -116,8 +131,10 @@ class Step
       end
     end if step.dependencies
 
-    str = ""
-    str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str = []
+
+    indent = prov_indent(step, offset, input_dependencies)
+    str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
     step.dependencies.dup.tap{|l| 
       l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
@@ -126,10 +143,10 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
+        str.concat(prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep], input_dependencies.dup).split("\n"))
       else
         if expand_repeats
-          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task, input_dependencies[dep], input_dependencies.dup)))
         else
           info = dep.info  || {}
           status = info[:status] || :missing
@@ -138,13 +155,14 @@ class Step
           status = :unsync if status == :done and not Open.exist?(path)
           status = :notfound if status == :noinfo and not Open.exist?(path)
 
-          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
+          dep_indent = prov_indent(dep, offset+1, input_dependencies)
+          str << dep_indent + Log.color(Step.status_color(status), Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
         end
       end
     end if step.dependencies
 
-    str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+    str << indent + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
 
-    str
+    str * "\n"
   end
 end

data/lib/scout/workflow/step/status.rb

@@ -33,6 +33,10 @@ class Step
     Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
+  def self.clean(file)
+    Step.new(file).clean
+  end
+
 
   def recursive_clean
     dependencies.each do |dep|

data/lib/scout/workflow/step.rb

@@ -33,7 +33,9 @@ class Step
   def inputs
     @inputs ||= begin
                   if info_file && Open.exists?(info_file)
-                    info[:inputs]
+                    inputs = info[:inputs]
+                    NamedArray.setup(inputs, info[:input_names]) if inputs && info[:input_names]
+                    inputs
                   else
                     []
                   end
@@ -61,6 +63,10 @@ class Step
     @name ||= File.basename(@path)
   end
 
+  def short_path
+    [workflow.to_s, task_name, name] * "/"
+  end
+
   def clean_name
     return @id if @id
     return info[:clean_name] if info.include? :clean_name
@@ -122,9 +128,10 @@ class Step
       begin
         Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
           clear_info
+          input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
           merge_info :status => :start, :start => Time.now,
             :pid => Process.pid, :pid_hostname => Misc.hostname, 
-            :inputs => MetaExtension.purge(inputs), :type => type,
+            :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
             :dependencies => dependencies.collect{|d| d.path }
 
           @result = exec
@@ -278,9 +285,10 @@ class Step
   end
 
   def step(task_name)
+    task_name = task_name.to_sym
     dependencies.each do |dep|
-      return dep if dep.task_name == task_name
-      return dep if dep.overriden_task == task_name
+      return dep if dep.task_name && dep.task_name.to_sym == task_name
+      return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
       rec_dep = dep.step(task_name)
       return rec_dep if rec_dep
     end
@@ -302,4 +310,8 @@ class Step
   def task_signature
     [workflow.to_s, task_name] * "#"
   end
+
+  def alias?
+    task.alias? if task
+  end
 end

data/lib/scout/workflow/task/dependencies.rb

@@ -54,6 +54,7 @@ module Task
       if provided_inputs.include?(overriden = [workflow.name, task] * "#")
         dep = provided_inputs[overriden]
         dep = Step.new dep unless Step === dep
+        dep = dep.dup
         dep.type = workflow.tasks[task].type
         dep.overriden_task = task
         dep.overriden_workflow = workflow

data/lib/scout/workflow/task/inputs.rb

@@ -87,6 +87,8 @@ module Task
       if Path.is_filename?(value) 
         relative_file = save_file_input(value, directory)
         Open.write(input_file + ".as_file", relative_file)
+      elsif Step === value
+        Open.write(input_file + ".as_step", value.short_path)
       elsif type == :file
         relative_file = save_file_input(value, directory)
         Persist.save(relative_file, input_file, :file)
@@ -115,6 +117,9 @@ module Task
           value = Open.read(filename).strip
           value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
           inputs[name] = value
+        elsif filename.end_with?('.as_step')
+          value = Open.read(filename).strip
+          inputs[name] = Step.load value
         elsif (options &&  (options[:noload] || options[:stream] || options[:nofile]))
           inputs[name] = filename
         else

data/lib/scout/workflow/task.rb

@@ -45,6 +45,21 @@ module Task
     IndiferentHash.setup(provided_inputs)
     id = DEFAULT_NAME if id.nil?
 
+
+    missing_inputs = []
+    self.inputs.each do |input,type,desc,val,options|
+      next unless options && options[:required]
+      missing_inputs << input unless provided_inputs.include?(input)
+    end if self.inputs
+
+    if missing_inputs.length == 1
+      raise ParameterException, "Input '#{missing_inputs.first}' is required but was not provided or is nil"
+    end
+
+    if missing_inputs.length > 1
+      raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
+    end
+
     provided_inputs = load_inputs(provided_inputs[:load_inputs]) if Hash === provided_inputs && provided_inputs[:load_inputs]
 
     inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
@@ -52,7 +67,7 @@ module Task
     compute = {}
     dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-    non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
+    #non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     non_default_inputs.uniq!
 
@@ -95,4 +110,8 @@ module Task
       Step.new path.find, inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
     end
   end
+
+  def alias?
+    @extension == :dep_task
+  end
 end

data/lib/scout/workflow/usage.rb

@@ -136,6 +136,7 @@ module Workflow
 
     dep_tree = {}
     task = self.tasks[task_name]
+    raise "TaskNotFound: #{task_name}" if task.nil?
     task.deps.each do |workflow, task, options|
       next if seen.include? dep
       seen << [workflow, task, options.merge(seen_options)]

data/lib/scout/workflow.rb

@@ -27,7 +27,9 @@ module Workflow
 
   def self.require_workflow(workflow_name_orig)
     first = nil
-    workflow_name_orig.split("+").each do |workflow_name|
+    workflow_name_orig.split("+").each do |complete_workflow_name|
+      self.main = nil
+      workflow_name, *subworkflows = complete_workflow_name.split("::")
       workflow = workflow_name
       workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
       workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
@@ -40,7 +42,14 @@ module Workflow
       else
         raise "Workflow #{workflow_name} not found"
       end
-      first ||= self.main || workflows.last
+
+      current = begin
+                  Kernel.const_get(complete_workflow_name)
+                rescue
+                  self.main || workflows.last
+                end
+
+      first ||= current
     end
     first
   end

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 9.0.0 ruby lib
+# stub: scout-gear 9.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "9.0.0"
+  s.version = "9.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-06-27"
+  s.date = "2023-06-30"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]

data/scout_commands/workflow/task

@@ -18,6 +18,7 @@ $ #{$0} [<options>] <workflow> <task>
 -h--help Print this help
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
+--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 -jn--jobname* Name to use as job identifier
 -li--load_inputs* Directory with inputs files to load
 -pf--print_filepath Print the file path
@@ -25,7 +26,6 @@ $ #{$0} [<options>] <workflow> <task>
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
 -ct--clean_task* Clean a particular task
--d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 EOF
 
@@ -86,8 +86,7 @@ if clean_task
       dep.set_info :status, :cleaned
     end
 
-    job.clean if job.task_name.to_s == clean_task.to_s
-    job.clean unless job.updated?
+    job.clean if (job.task_name.to_s == clean_task.to_s) || ! job.updated?
   end
 end
 

data/test/scout/resource/test_produce.rb

@@ -74,4 +74,19 @@ end
       assert_equal list, file.list
     end
   end
+
+  def test_produce_compressed
+    dir = tmpdir.directory[__method__]
+    dir.pkgdir = Scout
+    list = %w(a b)
+
+    Scout.claim dir["foo.gz"], :proc do |filename|
+      list
+    end
+
+    Misc.in_dir(dir) do
+      file = dir.foo
+      assert_include file.produce.find, '.gz'
+    end
+  end
 end

data/test/scout/resource/test_util.rb

@@ -7,6 +7,15 @@ class TestResourceUtil < Test::Unit::TestCase
     assert_equal 'share/data/somedir/somepath', Scout.identify(path)
   end
 
+  def test_identify_with_subdir
+    m = Module.new
+    m.extend Resource
+    m.subdir = 'test/subdir'
+    path = m.root['share/data/somedir/somepath'].find
+    assert_equal 'share/data/somedir/somepath', m.identify(path)
+  end
+
+
   def test_relocate
     TmpFile.with_file do |dir|
       Path.setup dir

data/test/scout/workflow/step/test_provenance.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/workflow'
 class TestStepProvenance < Test::Unit::TestCase
-  def test_true
+  def test_prov_report
     tmpfile = tmpdir.test_step
     step1 = Step.new tmpfile.step1, ["12"] do |s|
       s.length
@@ -21,5 +21,28 @@ class TestStepProvenance < Test::Unit::TestCase
     assert_include Step.prov_report(step2), 'step1'
     assert_include Step.prov_report(step2), 'step2'
   end
+
+  def __test_input_dependencies
+    wf = Workflow.annonymous_workflow "TaskInputDepProv" do
+      input :value1, :integer
+      task :number1 => :integer do |i| i end
+
+      input :value2, :integer
+      task :number2 => :integer do |i| i end
+
+      input :number1, :integer
+      input :number2, :integer
+      task :sum => :integer do |i1,i2| i1 + i2 end
+
+      dep :number1, :value1 => 1
+      dep :number2, :value1 => 2
+      dep_task :my_sum, self, :sum, :number1 => :number1, :number2 => :number2
+
+    end
+    job = wf.job(:my_sum)
+
+    ppp Step.prov_report(job)
+
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 9.0.0
+  version: 9.1.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-27 00:00:00.000000000 Z
+date: 2023-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor