scout-gear 7.1.0 → 7.2.0

data/lib/scout/workflow/step/provenance.rb

@@ -0,0 +1,148 @@
+class Step
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
+  def self.status_color(status)
+    case status.to_sym
+    when :error, :aborted, :missing, :dead, :unsync
+      :red
+    when :streaming, :started
+      :cyan
+    when :done, :noinfo
+      :green
+    when :dependencies, :waiting, :setup
+      :yellow
+    when :notfound, :cleaned
+      :blue
+    else
+      if status.to_s.index ">"
+        :cyan
+      else
+        :cyan
+      end
+    end
+  end
+
+  def self.prov_status_msg(status)
+    color = status_color(status)
+    Log.color(color, status.to_s)
+  end
+
+  def self.prov_report_msg(status, name, path, info, input = nil)
+    parts = path.sub(/\{.*/,'').split "/"
+
+    parts.pop
+
+    task = Log.color(:yellow, parts.pop)
+    workflow = Log.color(:magenta, parts.pop)
+
+    if ! Step.job_path?(path)
+      task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
+    end
+
+    path_mtime = begin
+                   Open.mtime(path)
+                 rescue Exception
+                   nil
+                 end
+
+    if input.nil? || input.empty?
+      input_str = nil
+    else
+      input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
+      input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
+      input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
+    end
+
+    str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+          else
+            prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " 
+          end
+
+    if $inputs and $inputs.any? 
+      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      IndiferentHash.setup(job_inputs)
+
+      $inputs.each do |input|
+        value = job_inputs[input]
+        next if  value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, input}=#{value_str}"
+      end
+    end
+
+    if $info_fields and $info_fields.any?
+      $info_fields.each do |field|
+        IndiferentHash.setup(info)
+        value = info[field]
+        next if value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, field}=#{value_str}"
+      end
+    end
+
+    str << "\n"
+  end
+
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
+    info = step.info  || {}
+    info[:task_name] = task
+    path  = step.path
+    status = info[:status] || :missing
+    status = "remote" if Open.remote?(path) || Open.ssh?(path)
+    name = info[:name] || File.basename(path)
+    status = :unsync if status == :done and not Open.exist?(path)
+    status = :notfound if status == :noinfo and not Open.exist?(path)
+
+
+    this_step_msg = prov_report_msg(status, name, path, info, input)
+
+    input_dependencies = {}
+    step.dependencies.each do |dep|
+      if dep.input_dependencies.any?
+        dep.input_dependencies.each do |id|
+          input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
+            d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
+          }.last
+          if input_name
+            input_dependencies[id] ||= []
+            input_dependencies[id] << [dep, input_name]
+          end
+        end
+      end
+    end if step.dependencies
+
+    str = ""
+    str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    step.dependencies.dup.tap{|l| 
+      l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    }.each do |dep|
+      path = dep.path
+      new = ! seen.include?(path)
+      if new
+        seen << path
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
+      else
+        if expand_repeats
+          str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
+        else
+          info = dep.info  || {}
+          status = info[:status] || :missing
+          status = "remote" if Open.remote?(path) || Open.ssh?(path)
+          name = info[:name] || File.basename(path)
+          status = :unsync if status == :done and not Open.exist?(path)
+          status = :notfound if status == :noinfo and not Open.exist?(path)
+
+          str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
+        end
+      end
+    end if step.dependencies
+
+    str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
+
+    str
+  end
+end

data/lib/scout/workflow/step.rb

@@ -2,15 +2,25 @@ require_relative '../path'
 require_relative '../persist'
 require_relative 'step/info'
 require_relative 'step/load'
+require_relative 'step/file'
+require_relative 'step/dependencies'
+require_relative 'step/provenance'
+require_relative 'step/config'
 
 class Step 
 
-  attr_accessor :path, :inputs, :dependencies, :task
+  attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
   def initialize(path, inputs = nil, dependencies = nil, &task) 
     @path = path
     @inputs = inputs
     @dependencies = dependencies
     @task = task
+    @mutex = Mutex.new
+    @tee_copies = 1
+  end
+
+  def synchronize(&block)
+    @mutex.synchronize(&block)
   end
 
   def inputs
@@ -48,30 +58,49 @@ class Step
     @task_name ||= @task.name if @task.respond_to?(:name)
   end
 
+  def workflow
+    @task.workflow if @task
+  end
+
   def exec
-    self.instance_exec(*inputs, &task)
+    @result = self.instance_exec(*inputs, &task)
   end
 
   attr_reader :result
   def run
     return @result || self.load if done?
-    dependencies.each{|dep| dep.run }
-    @result = Persist.persist(name, type, :path => path) do
+    prepare_dependencies
+    run_dependencies
+    @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
       begin
         merge_info :status => :start, :start => Time.now,
           :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"], 
           :inputs => inputs, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
-        @result = exec
+        exec
+      rescue Exception => e
+        merge_info :status => :error, :exception => e
+        raise e
       ensure
-        if streaming?
-          ConcurrentStream.setup(@result) do
+        if ! (error? || aborted?)
+          if streaming?
+            ConcurrentStream.setup(@result) do
+              merge_info :status => :done, :end => Time.now
+            end
+
+            @result.abort_callback = proc do |exception|
+              if Aborted === exception || Interrupt === exception
+                merge_info :status => :aborted, :end => Time.now
+              else
+                merge_info :status => :error, :exception => exception, :end => Time.now
+              end
+            end
+
+            log :streaming
+          else
             merge_info :status => :done, :end => Time.now
           end
-          log :streaming
-        else
-          merge_info :status => :done, :end => Time.now
         end
       end
     end
@@ -82,19 +111,34 @@ class Step
   end
 
   def streaming?
-    IO === @result || StringIO === @result
+    @take_stream || IO === @result || StringIO === @result
   end
 
-  def stream
-    join
-    streaming? ? @result : Open.open(path)
+  def get_stream
+    synchronize do
+      if streaming? && ! @result.nil?
+        if @result.next
+          Log.debug "Taking result #{Log.fingerprint @result} next #{Log.fingerprint @result.next}"
+        else
+          Log.debug "Taking result #{Log.fingerprint @result}"
+        end
+        @take_stream, @result = @result, @result.next
+        @take_stream
+      elsif done?
+        Open.open(self.path)
+      else
+        if running?
+          nil
+        else
+          exec
+        end
+      end
+    end
   end
 
   def join
-    if streaming?
-      Open.consume_stream(@result, false) 
-      @result = nil
-    end
+    stream = get_stream if streaming?
+    Open.consume_stream(stream, false) if stream
   end
 
   def produce
@@ -109,8 +153,13 @@ class Step
   end
 
   def clean
+    @take_stream = nil 
+    @result = nil
+    @info = nil
+    @info_load_time = nil
     Open.rm path if Open.exist?(path)
     Open.rm info_file if Open.exist?(info_file)
+    Open.rm_rf files_dir if Open.exist?(files_dir)
   end
 
   def recursive_clean
@@ -128,6 +177,6 @@ class Step
   end
 
   def digest_str
-    path
+    path.dup
   end
 end

data/lib/scout/workflow/task.rb

@@ -4,7 +4,7 @@ require_relative 'task/inputs'
 
 module Task
   extend MetaExtension
-  extension_attr :name, :type, :inputs, :deps, :directory, :description
+  extension_attr :name, :type, :inputs, :deps, :directory, :description, :returns, :extension, :workflow
 
   DEFAULT_NAME = "Default"
 
@@ -54,7 +54,7 @@ module Task
       resolved_inputs = {}
       inputs.each do |k,v|
         if Symbol === v
-          input_dep = dependencies.select{|d| d.task_name == v}.first
+          input_dep = dependencies.select{|d| d.task_name == v }.first
           resolved_inputs[k] = input_dep || inputs[v] || k
         else
           resolved_inputs[k] = v
@@ -150,6 +150,7 @@ module Task
 
     path = directory[id]
 
+    NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
     Step.new path.find, inputs, dependencies, &self
   end
 end

data/lib/scout/workflow/usage.rb

@@ -34,7 +34,6 @@ module Task
         str.puts Log.color :yellow, dep + ":"
         str.puts
         str.puts SOPT.input_array_doc(inputs)
-        str.puts
       end
 
       #task_inputs = dep_inputs deps, workflow
@@ -72,6 +71,7 @@ module Task
       str.puts
     end
 
+    str.puts
     str.puts Log.color(:magenta, "Returns: ") << Log.color(:blue, type.to_s) << "\n"
     str.puts
 

data/lib/scout/workflow.rb

@@ -24,17 +24,25 @@ module Workflow
     base.libdir = Path.setup(libdir).tap{|p| p.resource = base}
   end
 
-  def self.require_workflow(workflow)
-    workflow = Path.setup('workflows')[workflow]["workflow.rb"] unless Open.exists?(workflow)
+  def self.require_workflow(workflow_name)
+    workflow = workflow_name 
+    workflow = Path.setup('workflows')[workflow_name]["workflow.rb"] unless Open.exists?(workflow)
+    workflow = Path.setup('workflows')[Misc.snake_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
+    workflow = Path.setup('workflows')[Misc.camel_case(workflow_name)]["workflow.rb"] unless Open.exists?(workflow)
     if Open.exists?(workflow)
       workflow = workflow.find if Path === workflow
+      $LOAD_PATH.unshift(File.join(File.dirname(workflow), 'lib'))
       load workflow
+    else
+      raise "Workflow #{workflow_name} not found"
     end
     workflows.last
   end
 
   def job(name, *args)
     task = tasks[name]
-    task.job(*args)
+    step = task.job(*args)
+    step.extend step_module
+    step
   end
 end

data/lib/scout-gear.rb

@@ -7,3 +7,4 @@ require_relative 'scout/simple_opt'
 require_relative 'scout/resource'
 require_relative 'scout/resource/scout'
 require_relative 'scout/persist'
+require_relative 'scout/tsv'

data/lib/scout.rb

@@ -1 +1,2 @@
 require 'workflow-scout'
+require 'rbbt-scout'

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 7.1.0 ruby lib
+# stub: scout-gear 7.2.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "7.1.0"
+  s.version = "7.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-05-01"
+  s.date = "2023-05-09"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -28,10 +28,12 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/scout",
+    "lib/rbbt-scout.rb",
     "lib/scout-gear.rb",
     "lib/scout.rb",
     "lib/scout/cmd.rb",
     "lib/scout/concurrent_stream.rb",
+    "lib/scout/config.rb",
     "lib/scout/exceptions.rb",
     "lib/scout/indiferent_hash.rb",
     "lib/scout/indiferent_hash/case_insensitive.rb",
@@ -50,6 +52,8 @@ Gem::Specification.new do |s|
     "lib/scout/misc/format.rb",
     "lib/scout/misc/insist.rb",
     "lib/scout/misc/monitor.rb",
+    "lib/scout/misc/system.rb",
+    "lib/scout/named_array.rb",
     "lib/scout/open.rb",
     "lib/scout/open/lock.rb",
     "lib/scout/open/remote.rb",
@@ -68,6 +72,7 @@ Gem::Specification.new do |s|
     "lib/scout/resource/produce.rb",
     "lib/scout/resource/produce/rake.rb",
     "lib/scout/resource/scout.rb",
+    "lib/scout/resource/software.rb",
     "lib/scout/resource/util.rb",
     "lib/scout/semaphore.rb",
     "lib/scout/simple_opt.rb",
@@ -78,7 +83,15 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
     "lib/scout/tsv.rb",
+    "lib/scout/tsv/dumper.rb",
+    "lib/scout/tsv/index.rb",
     "lib/scout/tsv/parser.rb",
+    "lib/scout/tsv/path.rb",
+    "lib/scout/tsv/persist.rb",
+    "lib/scout/tsv/persist/adapter.rb",
+    "lib/scout/tsv/persist/tokyocabinet.rb",
+    "lib/scout/tsv/traverse.rb",
+    "lib/scout/tsv/util.rb",
     "lib/scout/work_queue.rb",
     "lib/scout/work_queue/socket.rb",
     "lib/scout/work_queue/worker.rb",
@@ -86,8 +99,12 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/step.rb",
+    "lib/scout/workflow/step/config.rb",
+    "lib/scout/workflow/step/dependencies.rb",
+    "lib/scout/workflow/step/file.rb",
     "lib/scout/workflow/step/info.rb",
     "lib/scout/workflow/step/load.rb",
+    "lib/scout/workflow/step/provenance.rb",
     "lib/scout/workflow/task.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
@@ -104,6 +121,7 @@ Gem::Specification.new do |s|
     "scout_commands/workflow/task_old",
     "share/color/color_names",
     "share/color/diverging_colors.hex",
+    "share/software/install_helpers",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
     "test/scout/log/test_color.rb",
@@ -111,6 +129,7 @@ Gem::Specification.new do |s|
     "test/scout/misc/test_digest.rb",
     "test/scout/misc/test_filesystem.rb",
     "test/scout/misc/test_insist.rb",
+    "test/scout/misc/test_system.rb",
     "test/scout/open/test_lock.rb",
     "test/scout/open/test_remote.rb",
     "test/scout/open/test_stream.rb",
@@ -122,6 +141,7 @@ Gem::Specification.new do |s|
     "test/scout/persist/test_serialize.rb",
     "test/scout/resource/test_path.rb",
     "test/scout/resource/test_produce.rb",
+    "test/scout/resource/test_software.rb",
     "test/scout/resource/test_util.rb",
     "test/scout/simple_opt/test_doc.rb",
     "test/scout/simple_opt/test_get.rb",
@@ -129,10 +149,12 @@ Gem::Specification.new do |s|
     "test/scout/simple_opt/test_setup.rb",
     "test/scout/test_cmd.rb",
     "test/scout/test_concurrent_stream.rb",
+    "test/scout/test_config.rb",
     "test/scout/test_indiferent_hash.rb",
     "test/scout/test_log.rb",
     "test/scout/test_meta_extension.rb",
     "test/scout/test_misc.rb",
+    "test/scout/test_named_array.rb",
     "test/scout/test_open.rb",
     "test/scout/test_path.rb",
     "test/scout/test_persist.rb",
@@ -142,11 +164,20 @@ Gem::Specification.new do |s|
     "test/scout/test_tsv.rb",
     "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
+    "test/scout/tsv/persist/test_adapter.rb",
+    "test/scout/tsv/persist/test_tokyocabinet.rb",
+    "test/scout/tsv/test_dumper.rb",
+    "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_parser.rb",
+    "test/scout/tsv/test_persist.rb",
+    "test/scout/tsv/test_traverse.rb",
+    "test/scout/tsv/test_util.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
+    "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
+    "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",

data/scout_commands/find

@@ -69,7 +69,7 @@ end if resource
 
 path = (resource || Scout)[path.dup]
 
-if where.nil? || where == 'all' || path.search_paths.include?(where.to_sym)
+if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
   location = path.find(where)
 
   if Array === location

data/scout_commands/workflow/task

@@ -13,6 +13,7 @@ $ #{$0} [<options>] <workflow> <task>
 -h--help Print this help
 -jn--job_name* Name to use as job identifier
 -pf--print_filepath Print the file path
+-prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
 EOF
@@ -22,7 +23,8 @@ workflow_name, task_name = ARGV
 raise MissingParameterException.new :workflow if workflow_name.nil?
 
 workflow = Workflow.require_workflow workflow_name
-task = workflow.tasks[task_name.to_sym] if task_name
+task_name = task_name.to_sym if task_name
+task = workflow.tasks[task_name.to_sym] 
 
 options[:help] = true if task.nil?
 
@@ -38,20 +40,24 @@ if options[:help]
 end
 
 job_options = task.get_SOPT(task)
-job = task.job(options[:job_name], job_options)
+job = workflow.job(task_name, options[:job_name], job_options)
 
 job.recursive_clean if options[:recursive_clean]
 job.clean if options[:clean]
 
-job.run
-
-if options[:print_filepath]
-  path = job.path
-  path = path.find if Path === path
-  puts path
+if options[:provenance]
+  puts Step.prov_report(job)
 else
-  if ! Open.consume_stream(job.stream, false, STDOUT, false).end_with? "\n"
-    puts
+  job.run unless job.done?
+
+  if options[:print_filepath]
+    path = job.path
+    path = path.find if Path === path
+    puts path
+  else
+    if ! ((c = Open.consume_stream(job.get_stream, false, STDOUT, false)) && c.end_with?("\n"))
+      puts
+    end
   end
 end