scout-gear 10.8.3 → 10.9.0

data/scout_commands/system/status

@@ -0,0 +1,238 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Report the status of the system
+
+$ rbbt system status <workflow> <task>
+
+Specify workflow '.' and no task to examine the jobs of the current directory (usefull for web-server cache).
+
+-h--help Print this help
+-q--quick Quick check
+-a--all Print all jobs, not only uncompleted
+-i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
+EOF
+scout_usage and exit 0 if options[:help]
+
+workflow, task = ARGV
+
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+
+inputs = (options[:inputs] || "").split(",")
+info_fields = (options[:info_fields] || "").split(",")
+
+def pid_msg(pid)
+  color = if pid and Misc.pid_alive? pid
+            :green
+          else
+            :red
+          end
+  if pid.nil?
+    ""
+  else
+    Log.color(color, pid)
+  end
+end
+
+
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
+
+def input_msg(file, inputs)
+
+  str = ""
+  job_inputs = Step.new(file).recursive_inputs.to_hash
+  IndiferentHash.setup(job_inputs)
+
+  inputs.each do |input|
+    value = job_inputs[input]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, input}=#{value_str}"
+  end
+  str
+end
+
+def info_msg(info, info_fields)
+
+  str = ""
+  info_fields.each do |field|
+    value = info[field]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, field}=#{value_str}"
+  end
+  str
+end
+
+
+puts Log.color(:magenta, "# System report")
+puts
+sort_files = Proc.new do |a,b|
+  fa,ia = a
+  fb,ib = b
+
+  dira = fa.split(/[:_][^\/>]*$/).first
+  dirb = fb.split(/[:_][^\/>]*$/).first
+
+  case dira <=> dirb
+  when -1
+    -1
+  when 1
+    1
+  else
+    ia[:ctime] <=> ib[:ctime]
+  end
+end
+
+
+persists = Scout.persist_info
+if persists.any?
+  puts Log.color(:magenta, "Persist:")
+  persists.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+sensiblewrites = Scout.sensiblewrite_info
+if sensiblewrites.any?
+  puts Log.color(:magenta, "Writing:")
+  sensiblewrites.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+locks = Scout.lock_info
+if locks.any?
+  puts Log.color(:magenta, "Locks:")
+  locks.sort(&sort_files).each do |file,info|
+    elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  end
+  puts
+end
+
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
+puts Log.color(:magenta, "# Workflows")
+
+if workflow === ['.']
+  jobs = Scout.job_info ["all"], task, ['.']
+else
+  jobs = Scout.job_info workflow, task
+end
+
+workflows = {}
+
+TSV.traverse jobs, :_bar => "Checking job status" do |file,info|
+  next unless all || ! info[:done] || ! File.exist?(file)
+  workflow = info[:workflow]
+  task = info[:task]
+  workflows[workflow] ||= {}
+  workflows[workflow][task] ||= {}
+  workflows[workflow][task][file] ||= info
+end
+
+workflows.sort.each do |workflow,tasks|
+  tasks.sort.each do |task,jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": " <<  Log.color(:blue, jobs.length.to_s)
+
+    files_txt = jobs.collect do |file, i|
+      str = file.dup
+      if options[:quick] and i[:done]
+        status = 'done'
+        str << " #{ Step.prov_status_msg status }"
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          info = begin
+                   Open.open(i[:info_file]) do |f|
+                     Step.load_info(f)
+                   end
+                 rescue
+                   Log.exception $!
+                   {:status => :noinfo}
+                 end
+          IndiferentHash.setup(info)
+          str << info_msg(info, info_fields)
+        end
+      else
+        info = begin
+                 Open.open(i[:info_file]) do |f|
+                   Step.load_info(f)
+                 end
+               rescue
+                 {:status => :noinfo}
+               end
+        IndiferentHash.setup(info)
+
+        pid = info[:pid]
+        status = info[:status]
+        status = :missing if status == :done and not (Open.exist?(file) && ! Open.broken_link?(file))
+        status = :broken if Open.broken_link?(file)
+        status = status.to_s
+        if status != "done" and pid and not Misc.pid_alive?(pid)
+          if File.exist? file
+            status << Log.color(:red, " (out of sync)") 
+          else
+            status << Log.color(:red, " (dead)") 
+          end
+        end
+        str << " #{ Step.prov_status_msg status }"
+        str << " (dirty)" if status == 'done' && Step.new(file).dirty?
+
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          str << info_msg(info, info_fields)
+        end
+      end
+      str << "; #{pid_msg pid}" unless status == "done"
+      str
+    end
+
+    files_txt.each do |f|
+      next if f.nil?
+      puts "  " << f
+    end
+  end
+end

data/scout_commands/workflow/info

@@ -11,6 +11,8 @@ Show info from job
 $ #{$0} [<options>] <step_path>
 
 -h--help Print this help
+-i--inputs Pretty print the inputs
+-ri--recursive_inputs Pretty print the inputs (recursively)
 EOF
 if options[:help]
   if defined? scout_usage
@@ -25,17 +27,28 @@ path = ARGV.first
 raise MissingParameterException.new :step_path if path.nil?
 step = Step.load(path.dup)
 
-step.info.each do |k,v|
-  case v
-  when nil
+if options[:inputs]
+  names, values = step.info.values_at :input_names, :inputs
+  names.zip(values).each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end
+elsif options[:recursive_inputs]
+  step.recursive_inputs.each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end
+else
+  step.info.each do |k,v|
+    case v
+    when nil
+      next
+    when Exception
+      puts Log.color(:title, "Exception")
+      Log.exception v
+    else
+      puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
+    end
+  rescue
     next
-  when Exception
-    puts Log.color(:title, "Exception")
-    Log.exception v
-  else
-    puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
   end
-rescue
-  next
 end
 

data/scout_commands/workflow/install

@@ -65,7 +65,7 @@ Misc.in_dir(workflow_dir) do
       Log.info "Installing: " + workflow
 
       if base_repo.nil?
-        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+        repo_base_url = Scout.etc.workflow_repo.exists? ? Scout.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
       else
         repo_base_url = base_repo
       end

data/scout_commands/workflow/task

@@ -115,7 +115,7 @@ else
   when "serial"
     job.run(true) 
   when "local"
-    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 3
+    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 0.5
     orchestrator = Workflow::Orchestrator.new refresh.to_f, "cpus" => Misc.processors
     orchestrator.process({}, job)
   when "slurm"

data/test/scout/entity/test_property.rb

@@ -335,7 +335,7 @@ class TestEntityProperty < Test::Unit::TestCase
 
   def test_all_properties
     assert ReversableString.setup("TEST").all_properties.include?(:reverse_text_ary)
-    assert_equal ReversableString.setup("TEST").all_properties, ReversableString.properties
+    assert_equal ReversableString.setup("TEST").all_properties.sort, ReversableString.properties.keys.sort
   end
 
   def test_times

data/test/scout/knowledge_base/test_registry.rb

@@ -31,5 +31,24 @@ Isa,IV
       end
     end
   end
+
+  def test_registry_identifiers_target
+    identifier =<<-EOF
+#Alias,Initials
+Clei,CC
+Miki,MV
+Guille,GC
+Isa,IV
+    EOF
+    TmpFile.with_dir do |dir|
+      TmpFile.with_file(identifier) do |identifier_file|
+        identifiers = TSV.open(identifier_file, sep: ",", type: :single)
+        brothers = datafile_test(:person).brothers
+        kb = KnowledgeBase.new dir
+        kb.register :brothers, brothers, identifiers: identifiers
+        assert_include kb.get_index(:brothers, source: "=>Initials", target: "=>Initials"), "CC~GC"
+      end
+    end
+  end
 end
 

data/test/scout/test_work_queue.rb

@@ -142,7 +142,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
       res = []
       q.process do |out|
-        raise ScoutException 
+        raise ScoutException
         res << out
       end
 

data/test/scout/work_queue/test_worker.rb

@@ -156,7 +156,7 @@ class TestQueueWorker < Test::Unit::TestCase
     input = WorkQueue::Socket.new
     output = WorkQueue::Socket.new
 
-    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers = 5.times.collect{ WorkQueue::Worker.new }
     workers.each do |w|
       w.process(input, output) do |obj|
         raise ScoutException
@@ -164,18 +164,19 @@ class TestQueueWorker < Test::Unit::TestCase
       end
     end
 
-    read = Thread.new do 
+    Open.purge_pipes(input.swrite, output.sread)
+    read = Thread.new do
       Thread.current.report_on_exception = false
-      begin
-        while obj = output.read
-          if DoneProcessing === obj
-            pid = obj.pid
-            @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
-            break if workers.empty?
-          end
-          raise obj if Exception === obj
+      while obj = output.read
+        if DoneProcessing === obj
+          pid = obj.pid
+          @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
+          break if workers.empty?
         end
+        raise obj if Exception === obj
       end
+    ensure
+      output.close_read
     end
 
     write = Thread.new do
@@ -187,6 +188,7 @@ class TestQueueWorker < Test::Unit::TestCase
         input.write DoneProcessing.new
       end
       input.close_write
+    rescue
     end
 
     write.join

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.8.3
+  version: 10.9.0
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-06-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -65,6 +65,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: juwelier
   requirement: !ruby/object:Gem::Requirement
@@ -87,18 +101,26 @@ executables:
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
+- README.md
 - README.rdoc
 files:
 - ".document"
 - ".gitmodules"
 - ".vimproject"
 - LICENSE.txt
+- README.md
 - README.rdoc
 - Rakefile
 - VERSION
 - bin/scout
-- doc/lib/scout/path.md
-- doc/lib/scout/workflow/task.md
+- doc/Association.md
+- doc/Entity.md
+- doc/KnowledgeBase.md
+- doc/Persist.md
+- doc/Semaphore.md
+- doc/TSV.md
+- doc/WorkQueue.md
+- doc/Workflow.md
 - lib/scout-gear.rb
 - lib/scout.rb
 - lib/scout/association.rb
@@ -120,6 +142,7 @@ files:
 - lib/scout/knowledge_base/query.rb
 - lib/scout/knowledge_base/registry.rb
 - lib/scout/knowledge_base/traverse.rb
+- lib/scout/monitor.rb
 - lib/scout/persist/engine.rb
 - lib/scout/persist/engine/fix_width_table.rb
 - lib/scout/persist/engine/packed_index.rb
@@ -198,7 +221,9 @@ files:
 - scout_commands/alias
 - scout_commands/batch/clean
 - scout_commands/batch/list
+- scout_commands/cat
 - scout_commands/doc
+- scout_commands/entity
 - scout_commands/find
 - scout_commands/glob
 - scout_commands/kb/config
@@ -212,6 +237,8 @@ files:
 - scout_commands/rbbt
 - scout_commands/resource/produce
 - scout_commands/resource/sync
+- scout_commands/system/clean
+- scout_commands/system/status
 - scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/cmd
@@ -334,7 +361,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: basic gear for scouts
 test_files: []

data/doc/lib/scout/path.md

@@ -1,35 +0,0 @@
-Path
-===
-
-```ruby
-:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
-:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
-:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
-:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:default => :user
-
-@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
-
-```
-
-
-
-# tags
-{PKGDIR}
-{LIBDIR}
-{RESOURCE}
-{HOME}
-{PWD}
-{TOPLEVEL}
-{SUBPATH}
-{BASENAME}
-{PATH}
-{MAPNAME}
-{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -1,13 +0,0 @@
-Task
-====
-
-```ruby
-    wf = Workflow.annonymous_workflow "TaskInputs" do
-      input :input1, :integer
-      task :step1 => :integer do |i| i end
-
-      dep :step1
-      input :input2, :integer, "Integer", 3
-      task :step2 => :integer do |i| i * step(:step1).load end
-    end
-```