scout-gear 10.8.4 → 10.10.1

data/scout_commands/system/status

@@ -0,0 +1,238 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Report the status of the system
+
+$ rbbt system status <workflow> <task>
+
+Specify workflow '.' and no task to examine the jobs of the current directory (usefull for web-server cache).
+
+-h--help Print this help
+-q--quick Quick check
+-a--all Print all jobs, not only uncompleted
+-i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
+EOF
+scout_usage and exit 0 if options[:help]
+
+workflow, task = ARGV
+
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+
+inputs = (options[:inputs] || "").split(",")
+info_fields = (options[:info_fields] || "").split(",")
+
+def pid_msg(pid)
+  color = if pid and Misc.pid_alive? pid
+            :green
+          else
+            :red
+          end
+  if pid.nil?
+    ""
+  else
+    Log.color(color, pid)
+  end
+end
+
+
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
+
+def input_msg(file, inputs)
+
+  str = ""
+  job_inputs = Step.new(file).recursive_inputs.to_hash
+  IndiferentHash.setup(job_inputs)
+
+  inputs.each do |input|
+    value = job_inputs[input]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, input}=#{value_str}"
+  end
+  str
+end
+
+def info_msg(info, info_fields)
+
+  str = ""
+  info_fields.each do |field|
+    value = info[field]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, field}=#{value_str}"
+  end
+  str
+end
+
+
+puts Log.color(:magenta, "# System report")
+puts
+sort_files = Proc.new do |a,b|
+  fa,ia = a
+  fb,ib = b
+
+  dira = fa.split(/[:_][^\/>]*$/).first
+  dirb = fb.split(/[:_][^\/>]*$/).first
+
+  case dira <=> dirb
+  when -1
+    -1
+  when 1
+    1
+  else
+    ia[:ctime] <=> ib[:ctime]
+  end
+end
+
+
+persists = Scout.persist_info
+if persists.any?
+  puts Log.color(:magenta, "Persist:")
+  persists.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+sensiblewrites = Scout.sensiblewrite_info
+if sensiblewrites.any?
+  puts Log.color(:magenta, "Writing:")
+  sensiblewrites.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+locks = Scout.lock_info
+if locks.any?
+  puts Log.color(:magenta, "Locks:")
+  locks.sort(&sort_files).each do |file,info|
+    elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  end
+  puts
+end
+
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
+puts Log.color(:magenta, "# Workflows")
+
+if workflow === ['.']
+  jobs = Scout.job_info ["all"], task, ['.']
+else
+  jobs = Scout.job_info workflow, task
+end
+
+workflows = {}
+
+TSV.traverse jobs, :_bar => "Checking job status" do |file,info|
+  next unless all || ! info[:done] || ! File.exist?(file)
+  workflow = info[:workflow]
+  task = info[:task]
+  workflows[workflow] ||= {}
+  workflows[workflow][task] ||= {}
+  workflows[workflow][task][file] ||= info
+end
+
+workflows.sort.each do |workflow,tasks|
+  tasks.sort.each do |task,jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": " <<  Log.color(:blue, jobs.length.to_s)
+
+    files_txt = jobs.collect do |file, i|
+      str = file.dup
+      if options[:quick] and i[:done]
+        status = 'done'
+        str << " #{ Step.prov_status_msg status }"
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          info = begin
+                   Open.open(i[:info_file]) do |f|
+                     Step.load_info(f)
+                   end
+                 rescue
+                   Log.exception $!
+                   {:status => :noinfo}
+                 end
+          IndiferentHash.setup(info)
+          str << info_msg(info, info_fields)
+        end
+      else
+        info = begin
+                 Open.open(i[:info_file]) do |f|
+                   Step.load_info(f)
+                 end
+               rescue
+                 {:status => :noinfo}
+               end
+        IndiferentHash.setup(info)
+
+        pid = info[:pid]
+        status = info[:status]
+        status = :missing if status == :done and not (Open.exist?(file) && ! Open.broken_link?(file))
+        status = :broken if Open.broken_link?(file)
+        status = status.to_s
+        if status != "done" and pid and not Misc.pid_alive?(pid)
+          if File.exist? file
+            status << Log.color(:red, " (out of sync)") 
+          else
+            status << Log.color(:red, " (dead)") 
+          end
+        end
+        str << " #{ Step.prov_status_msg status }"
+        str << " (dirty)" if status == 'done' && Step.new(file).dirty?
+
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          str << info_msg(info, info_fields)
+        end
+      end
+      str << "; #{pid_msg pid}" unless status == "done"
+      str
+    end
+
+    files_txt.each do |f|
+      next if f.nil?
+      puts "  " << f
+    end
+  end
+end

data/scout_commands/workflow/cmd

@@ -12,20 +12,12 @@ $ #{$0} <workflow> <command> [<subcommands>] [<options>] [<arg> ...]
 
 -h--help Print this help
 EOF
-if options[:help]
-  if defined? scout_usage
-    scout_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
-end
 
 workflow = ARGV.shift
 
 if workflow == '-h'
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -99,11 +91,11 @@ puts
 puts Log.color :magenta, "## COMMANDS"
 puts
 puts Log.color :magenta, "Command:"
-puts 
+puts
 puts " scout #{$previous_commands * " "} "
-puts 
+puts
 puts Log.color :magenta, "Subcommands:"
-puts 
+puts
 
 commands(prev).each do |command|
   directory = File.directory? dir[command].find
@@ -113,4 +105,4 @@ commands(prev).each do |command|
     puts "  " << command
   end
 end
-    
+

data/scout_commands/workflow/info

@@ -11,6 +11,8 @@ Show info from job
 $ #{$0} [<options>] <step_path>
 
 -h--help Print this help
+-i--inputs Pretty print the inputs
+-ri--recursive_inputs Pretty print the inputs (recursively)
 EOF
 if options[:help]
   if defined? scout_usage
@@ -25,17 +27,28 @@ path = ARGV.first
 raise MissingParameterException.new :step_path if path.nil?
 step = Step.load(path.dup)
 
-step.info.each do |k,v|
-  case v
-  when nil
+if options[:inputs]
+  names, values = step.info.values_at :input_names, :inputs
+  names.zip(values).each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end if names
+elsif options[:recursive_inputs]
+  step.recursive_inputs.each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end
+else
+  step.info.each do |k,v|
+    case v
+    when nil
+      next
+    when Exception
+      puts Log.color(:title, "Exception")
+      Log.exception v
+    else
+      puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
+    end
+  rescue
     next
-  when Exception
-    puts Log.color(:title, "Exception")
-    Log.exception v
-  else
-    puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
   end
-rescue
-  next
 end
 

data/scout_commands/workflow/install

@@ -65,7 +65,7 @@ Misc.in_dir(workflow_dir) do
       Log.info "Installing: " + workflow
 
       if base_repo.nil?
-        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+        repo_base_url = Scout.etc.workflow_repo.exists? ? Scout.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
       else
         repo_base_url = base_repo
       end

data/scout_commands/workflow/task

@@ -16,10 +16,13 @@ Run a workflow job
 $ #{$0} [<options>] <workflow> <task>
 
 -h--help Print this help
+--workflows* Other workflows to load
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
 --deploy* Deploy mode: serial, local, queue, or SLURM (default 'serial')
+--deploy_rules* Deployment rules; can be files of keywords
 --fork Fork and return path
+--exec Exec and return output
 --load_inputs* Directory or file with inputs files to load
 --save_inputs* Directory or tar.gz file path to store inputs
 -jn--jobname* Name to use as job identifier
@@ -33,33 +36,45 @@ EOF
 
 workflow_name, task_name = ARGV
 
+
+help, workflows, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, deploy_rules, override_deps, do_fork, do_exec = IndiferentHash.process_options options,
+  :help, :workflows, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :deploy_rules, :override_deps, :fork, :exec,
+  :deploy => 'serial'
+
 raise MissingParameterException.new :workflow if workflow_name.nil?
 
 workflow = Workflow.require_workflow workflow_name
+
+workflows = workflows.split(/,\s*/) if workflows
+workflows.each{|wf| Workflow.require_workflow wf } if workflows
+
 task_name = task_name.to_sym if task_name
 task = workflow.tasks[task_name.to_sym] if task_name
 
-options[:help] = true if task.nil?
-
-help, provenance, clean, recursive_clean, clean_task, load_inputs, save_inputs, jobname, printpath, deploy, override_deps, do_fork = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :save_inputs, :jobname, :printpath, :deploy, :override_deps, :fork,
-  :deploy => 'serial'
+help = true if task.nil?
 
 if help
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
 
   puts workflow.usage(task) if workflow
-  exit 0
+
+  if task_name && task.nil?
+    exit -1
+  else
+    exit 0
+  end
 end
 
+task = workflow.tasks[task_name]
+
 job_inputs = task.get_SOPT
 
 if load_inputs
-  job_inputs = job_inputs.merge(workflow.tasks[task_name].load_inputs(load_inputs))
+  job_inputs = job_inputs.merge(task.load_inputs(load_inputs))
 end
 
 if override_deps
@@ -99,10 +114,21 @@ elsif do_fork
   job.fork
   puts job.path
   exit 0
+elsif do_exec
+  puts job.exec
+  exit 0
 elsif save_inputs
   puts job.save_inputs(save_inputs)
   exit 0
 else
+  require 'scout/workflow/deployment'
+
+  if deploy_rules.nil?
+    rules = Workflow::Orchestrator.load_rules_for_job(job)
+  else
+    deploy_files = deploy_rules.split(/,\s*/)
+    rules = Workflow::Orchestrator.load_rules deploy_files
+  end
 
   case deploy
   when "queue"
@@ -112,28 +138,40 @@ else
       puts save_inputs
       exit
     end
+  when 'dry', 'dryrun', 'debug'
+    batches = Workflow::Orchestrator.job_batches rules, job
+    batches.each do |batch|
+      top = batch[:top_level]
+      jobs = batch[:jobs]
+      puts Log.color :title, Log.fingerprint(top)
+      puts batch[:rules].to_yaml
+      deps = batch[:deps].collect{|dep| dep[:top_level] }
+      provided_inputs = jobs.inject({}){|acc,j| acc = IndiferentHash.add_defaults acc, j.provided_inputs }
+      provided_inputs.each do |i,v|
+        puts Log.color :green, "#{i}: " + Log.color(:blue, Log.fingerprint(v))
+      end
+      other = jobs - [top]
+      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
+      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
+    end
+    exit 0
   when "serial"
     job.run(true) 
   when "local"
-    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 0.5
-    orchestrator = Workflow::Orchestrator.new refresh.to_f, "cpus" => Misc.processors
-    orchestrator.process({}, job)
-  when "slurm"
-    require 'rbbt-scout'
-    require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
-    slurm_options = {}
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, slurm_options)
-    job.grace
-  else 
-    if deploy.end_with?('-slurm')
-      server = deploy.sub('-slurm','')
-      OffsiteStep.setup(job, server: server, slurm: true)
+    Workflow::LocalExecutor.produce job, rules
+  when 'batch', 'sched', 'cluster', "slurm", 'pbs', 'lsf'
+    Workflow::Scheduler.produce job, rules
+    exit 0
+  else
+    raise 'Use scout-camp instead of scout' unless defined? OffsiteStep
+    if deploy.end_with?('-batch')
+      server = deploy.sub('-batch','')
+      OffsiteStep.setup(job, server: server, batch: true)
     else
       OffsiteStep.setup(job, server: deploy)
     end
 
-    job.run
+    job.run(true)
   end unless job.done?
 
   if printpath
@@ -147,5 +185,3 @@ else
     end
   end
 end
-
-

data/test/scout/entity/test_property.rb

@@ -335,7 +335,7 @@ class TestEntityProperty < Test::Unit::TestCase
 
   def test_all_properties
     assert ReversableString.setup("TEST").all_properties.include?(:reverse_text_ary)
-    assert_equal ReversableString.setup("TEST").all_properties, ReversableString.properties
+    assert_equal ReversableString.setup("TEST").all_properties.sort, ReversableString.properties.keys.sort
   end
 
   def test_times

data/test/scout/knowledge_base/test_registry.rb

@@ -31,5 +31,24 @@ Isa,IV
       end
     end
   end
+
+  def test_registry_identifiers_target
+    identifier =<<-EOF
+#Alias,Initials
+Clei,CC
+Miki,MV
+Guille,GC
+Isa,IV
+    EOF
+    TmpFile.with_dir do |dir|
+      TmpFile.with_file(identifier) do |identifier_file|
+        identifiers = TSV.open(identifier_file, sep: ",", type: :single)
+        brothers = datafile_test(:person).brothers
+        kb = KnowledgeBase.new dir
+        kb.register :brothers, brothers, identifiers: identifiers
+        assert_include kb.get_index(:brothers, source: "=>Initials", target: "=>Initials"), "CC~GC"
+      end
+    end
+  end
 end
 

data/test/scout/test_work_queue.rb

@@ -142,7 +142,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
       res = []
       q.process do |out|
-        raise ScoutException 
+        raise ScoutException
         res << out
       end
 

data/test/scout/work_queue/test_worker.rb

@@ -156,7 +156,7 @@ class TestQueueWorker < Test::Unit::TestCase
     input = WorkQueue::Socket.new
     output = WorkQueue::Socket.new
 
-    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers = 5.times.collect{ WorkQueue::Worker.new }
     workers.each do |w|
       w.process(input, output) do |obj|
         raise ScoutException
@@ -164,18 +164,19 @@ class TestQueueWorker < Test::Unit::TestCase
       end
     end
 
-    read = Thread.new do 
+    Open.purge_pipes(input.swrite, output.sread)
+    read = Thread.new do
       Thread.current.report_on_exception = false
-      begin
-        while obj = output.read
-          if DoneProcessing === obj
-            pid = obj.pid
-            @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
-            break if workers.empty?
-          end
-          raise obj if Exception === obj
+      while obj = output.read
+        if DoneProcessing === obj
+          pid = obj.pid
+          @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
+          break if workers.empty?
         end
+        raise obj if Exception === obj
       end
+    ensure
+      output.close_read
     end
 
     write = Thread.new do
@@ -187,6 +188,7 @@ class TestQueueWorker < Test::Unit::TestCase
         input.write DoneProcessing.new
       end
       input.close_write
+    rescue
     end
 
     write.join