scout-gear 10.7.0 → 10.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac64d599e0df97b8ba5abd7aef4855ed1181ab9d9a3ba39196255b6649f56a0c
-  data.tar.gz: 9f1cf16da9432beb46eeb486de5bab1c57e460b09f2126556126eb356c6a6fd6
+  metadata.gz: 3e72b6d2bef9af0067bf2601e059aa992ffdab98fba772dba1e297f958edb02d
+  data.tar.gz: 2c2cc3591c1645ccf34dbb250f1b2acda1b69040c0b051806c02c97eb07cc253
 SHA512:
-  metadata.gz: 27015e493797ff3e80631a49e0d9bf2ccc115997e86d449b217dd55adc6dd0abd7701019c19467e9c48aa6bc7ff3c372238fcd58200069533686e2729870938f
-  data.tar.gz: 4930f226926fdc18b06952f037f8e60b45b49116be115409cbcee36bf66202f0bb909691f20fddc6711babbb2ec9b604a5f3e551742ab52cb3f96748665b13a6
+  metadata.gz: d1be99dd4127f54c5cf16833197e527a3cfa41014c7db4306ec291e8f01b78f8340cad7ee6e342dee2ed25221f2b02a6f6b665f46e81365ad5a12eea087b300d
+  data.tar.gz: cc3a9b28e47520f6756cba18d6037d7e83b7cd4d0cded1597a7cafc2cc167234090c1014d2b38adcc0a6a69c8d3ba1a72255538bd924e14f55b95769f3eca977

data/.vimproject

@@ -7,10 +7,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   scout=scout{
    workflow.rb
    workflow=workflow{
+    exceptions.rb
     definition.rb
     documentation.rb
     usage.rb
     util.rb
+    path.rb
     step.rb
     step=step{
      config.rb
@@ -80,7 +82,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      sort.rb
      unzip.rb
     }
-    entity.rb
     parser.rb
     dumper.rb
     transformer.rb

data/VERSION

@@ -1 +1 @@
-10.7.0
+10.7.1

data/lib/scout/entity.rb

@@ -34,6 +34,8 @@ module Entity
       entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) 
 
       entity = mod.setup(entity, params)
+
+      entity.extend AnnotatedArray if Array === entity && ! options[:annotated_array] == FalseClass
     end
 
     entity

data/lib/scout/tsv/parser.rb

@@ -352,6 +352,7 @@ module TSV
           all_field_names ||= [@key_field] + @fields
           fields = all_field_names if fields == :all
           positions = NamedArray.identify_name(all_field_names, fields)
+
           raise "Not all fields (#{Log.fingerprint fields}) identified in #{Log.fingerprint all_field_names}" if positions.include?(nil)
           kwargs[:positions] = positions
           field_names = all_field_names.values_at *positions

data/lib/scout/workflow/definition.rb

@@ -140,8 +140,17 @@ module Workflow
     end
   end
 
-  FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
-  REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
+  FORGET_TASK_ALIAS = begin 
+                        %w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
+                          ENV[var] == 'true'
+                        end.any?
+                      end
+  REMOVE_TASK_ALIAS = begin 
+                        remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
+                          ENV.include?(var) && ENV[var] != 'false'
+                        end.first
+                        remove.nil? ? false : remove
+                      end
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
@@ -158,9 +167,15 @@ module Workflow
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :type, dep.info[:type]
-      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
+
+      forget = config :forget_task_alias, "forget_task_alias"
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
+
       if forget
-        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_task_alias, "remove_task_alias"
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_TASK_ALIAS if remove.nil?
+
+        Log.medium "Forget task_alias (remove: #{remove}): #{short_path}"
 
         self.archive_deps
         self.copy_linked_files_dir
@@ -179,9 +194,9 @@ module Workflow
           when 'recursive'
             (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
               next if d.overriden
-              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+              d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
             end
-            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+            dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
           end 
         end
       else

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -253,7 +253,7 @@ module Workflow
     end
   end
 
-  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors)
+  def self.produce_dependencies(jobs, tasks, produce_cpus = Etc.nprocessors, produce_timer = 5)
     jobs = [jobs] unless Array === jobs
     produce_list = []
     jobs.each do |job|
@@ -265,7 +265,7 @@ module Workflow
       end
     end
 
-    orchestrator = Orchestrator.new 0.1, cpus: produce_cpus.to_i
+    orchestrator = Orchestrator.new produce_timer, cpus: produce_cpus.to_i
     orchestrator.process({}, produce_list)
     produce_list
   end

data/lib/scout/workflow/exceptions.rb

@@ -0,0 +1 @@
+class TaskNotFound < StandardError; end

data/lib/scout/workflow/path.rb

@@ -0,0 +1,40 @@
+module Path
+  def self.step_file?(path)
+    return false unless path =~ /\.files(?:\/|$)/
+    parts = path.split("/")
+    job = parts.select{|p| p =~ /\.files$/}.first
+
+    if job
+      i = parts.index job
+      begin
+        workflow, task = parts.values_at i - 2, i - 1
+        _loaded = false
+        begin
+          Kernel.const_get(workflow)
+        rescue 
+          if ! _loaded
+            Workflow.require_workflow workflow 
+            _loaded = true
+            retry
+          end
+          raise $!
+        end
+        return parts[i-2..-1] * "/"
+      rescue
+        Log.exception $!
+      end
+    end
+
+    false
+  end
+
+  alias original_digest_str digest_str
+
+  def digest_str
+    if step_file = Path.step_file?(self)
+      "Step file: #{step_file}"
+    else
+      original_digest_str
+    end
+  end
+end

data/lib/scout/workflow/step/dependencies.rb

@@ -11,7 +11,7 @@ class Step
   end
 
   def recursive_inputs
-    recursive_inputs = NamedArray === @inputs ? @inputs.to_hash : {}
+    recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
     return recursive_inputs if dependencies.nil?
     dependencies.inject(recursive_inputs) do |acc,dep|
       acc.merge(dep.recursive_inputs)

data/lib/scout/workflow/step/info.rb

@@ -10,7 +10,11 @@ class Step
   end
 
   def self.load_info(info_file)
-    info = Persist.load(info_file, SERIALIZER) || {}
+    info = begin
+             Persist.load(info_file, SERIALIZER) || {}
+           rescue
+             {status: :noinfo}
+           end
     IndiferentHash.setup(info)
   end
 
@@ -152,7 +156,7 @@ class Step
   end
 
   def running?
-    ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
+    ! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
 
   def overriden?

data/lib/scout/workflow/step/status.rb

@@ -31,6 +31,7 @@ class Step
   end
 
   def clean
+    Log.debug "Cleaning job files: #{path}"
     @take_stream = nil 
     @result = nil
     @info = nil

data/lib/scout/workflow/step.rb

@@ -372,7 +372,8 @@ class Step
   end
 
   def task_signature
-    [workflow.to_s, task_name] * "#"
+    workflow_name = String === workflow ? workflow : workflow.name
+    [workflow, task_name] * "#"
   end
 
   def alias?

data/lib/scout/workflow/task/inputs.rb

@@ -91,6 +91,8 @@ module Task
     if Path.is_filename?(value) 
       if type == :path
         Open.write(input_file + ".as_path", value)
+      elsif Path.step_file?(value)
+        Open.write(input_file + ".as_path", value)
       else
         relative_file = save_file_input(value, directory)
         Open.write(input_file + ".as_file", relative_file)

data/lib/scout/workflow/task.rb

@@ -92,7 +92,6 @@ module Task
         end
       end
 
-
       path = directory[name]
 
       path = path.set_extension(extension) if extension

data/lib/scout/workflow/usage.rb

@@ -138,7 +138,7 @@ module Workflow
 
     dep_tree = {}
     task = self.tasks[task_name]
-    raise "TaskNotFound: #{task_name}" if task.nil?
+    raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
     task.deps.each do |workflow, task, options|
       next if seen.include? dep
       seen << [workflow, task, options.merge(seen_options)]

data/lib/scout/workflow.rb

@@ -5,6 +5,8 @@ require_relative 'workflow/step'
 require_relative 'workflow/documentation'
 require_relative 'workflow/usage'
 require_relative 'workflow/deployment'
+require_relative 'workflow/exceptions'
+require_relative 'workflow/path'
 
 require 'scout/resource'
 require 'scout/resource/scout'
@@ -153,6 +155,7 @@ module Workflow
 
   def job(name, *args)
     task = tasks[name]
+    raise TaskNotFound, "Task #{task_name} in #{self.to_s}" if task.nil?
     step = task.job(*args)
     step.extend step_module
     step

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.7.0 ruby lib
+# stub: scout-gear 10.7.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.7.0".freeze
+  s.version = "10.7.1".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2024-06-05"
+  s.date = "2024-07-01"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -97,6 +97,8 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/deployment/orchestrator.rb",
     "lib/scout/workflow/deployment/trace.rb",
     "lib/scout/workflow/documentation.rb",
+    "lib/scout/workflow/exceptions.rb",
+    "lib/scout/workflow/path.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/archive.rb",
     "lib/scout/workflow/step/children.rb",
@@ -212,6 +214,7 @@ Gem::Specification.new do |s|
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
+    "test/scout/workflow/test_path.rb",
     "test/scout/workflow/test_step.rb",
     "test/scout/workflow/test_task.rb",
     "test/scout/workflow/test_usage.rb",

data/test/scout/workflow/test_definition.rb

@@ -66,5 +66,36 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     assert_include job.archived_info, dep_path
     assert_equal :done, job.archived_info[dep_path][:status]
   end
+
+  def test_task_alias_remove_dep_partial
+    wf = Workflow.annonymous_workflow do
+      self.name = "CallName"
+
+      task :salute => :string do |name|
+        "Hi"
+      end
+
+      dep :salute
+      input :name, :string, "Name to call", nil, :jobname => true
+      task :call_name => :string do |name|
+        "#{step(:salute).load} #{name}"
+      end
+
+      task_alias :call_miguel, self, :call_name, name: "Miguel"
+    end
+
+    old_cache = Scout::Config::CACHE.dup
+    Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => :recursive}, 'task:CallName#call_miguel')
+    Scout::Config.set({:remove_dep => false}, 'task:CallName#call_name')
+    job = wf.job(:call_miguel)
+    call_name = job.step(:call_name)
+    salute = job.step(:salute)
+    assert_equal "Hi Miguel", job.run
+    refute salute.done?
+    assert call_name.done?
+    Scout::Config::CACHE.replace old_cache
+    assert_include job.archived_info, call_name.path
+    assert_equal :done, job.archived_info[call_name.path][:status]
+  end
 end
 

data/test/scout/workflow/test_path.rb

@@ -0,0 +1,9 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_path_digest
+    iii Misc.digest_str(Path.setup("test_file"))
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.7.0
+  version: 10.7.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-06-05 00:00:00.000000000 Z
+date: 2024-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -209,6 +209,8 @@ files:
 - lib/scout/workflow/deployment/orchestrator.rb
 - lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
+- lib/scout/workflow/exceptions.rb
+- lib/scout/workflow/path.rb
 - lib/scout/workflow/step.rb
 - lib/scout/workflow/step/archive.rb
 - lib/scout/workflow/step/children.rb
@@ -324,6 +326,7 @@ files:
 - test/scout/workflow/task/test_inputs.rb
 - test/scout/workflow/test_definition.rb
 - test/scout/workflow/test_documentation.rb
+- test/scout/workflow/test_path.rb
 - test/scout/workflow/test_step.rb
 - test/scout/workflow/test_task.rb
 - test/scout/workflow/test_usage.rb