scout-gear 10.8.2 → 10.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a469436e6ab560158eead4f0fae044ebddebe65d611a97a69837ab7884cad3e4
-  data.tar.gz: a60763c026694b77083f23cf14784e4e6575bd34886980c4af28ead001f8590e
+  metadata.gz: 97c074aa2b85744eccbcba77c7be3eaa733d7280f6c5875211a3a1b771234e04
+  data.tar.gz: c284bf58c6954699fdc0b0742f6331ea1af7db7188b76b99325ad561e19de6cf
 SHA512:
-  metadata.gz: 5620d8f89b1c5b7bff57c21c2fb28bac8c31168220a7fd8b9804179e9381b75603e9844440e233f07c0d80ebc7de8f684235a34297263ca927679a4f08aa348c
-  data.tar.gz: b91fb51fec1e97b351a2d7a91b07186498b49d2b8bdf4561bb806b7b38cf45c2c2239642830bc1bfb9909b2f5ed7f173b3ce3707ca2f9fd799fd89d246217fa3
+  metadata.gz: f381c4c3167580d337ec5ac43bec9e12e8db0b4b423603fe134b8751d3a9aa8b9930797c87daa9cd0b7cf7f9c24c60ece3c00b2b4b70aec03aa71eae7bcef091
+  data.tar.gz: f39b067a52e6724302dc92d1eeb4dac3c22ce07e25c4e986d9a3018c92dcfdd79441d23bd2c9f0dca90a7a39d6ca95bc52fab92b0b4999cc003e48767e26bce8

data/.vimproject

@@ -5,6 +5,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  }
  lib=lib {
   scout=scout{
+   monitor.rb
    workflow.rb
    workflow=workflow{
     exceptions.rb
@@ -171,6 +172,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    produce
    sync
   }
+  system=system{
+   clean
+  }
  }
  test=test {
   test_helper.rb

data/Rakefile

@@ -24,6 +24,7 @@ Juwelier::Tasks.new do |gem|
   gem.add_runtime_dependency 'net-ssh'
   gem.add_runtime_dependency 'matrix'
   gem.add_runtime_dependency 'RubyInline'
+  gem.add_runtime_dependency 'csv'
   #gem.add_runtime_dependency 'tokyocabinet'
 
   gem.add_development_dependency "juwelier", "~> 2.1.0"

data/VERSION

@@ -1 +1 @@
-10.8.2
+10.8.4

data/lib/scout/monitor.rb

@@ -0,0 +1,296 @@
+require 'scout'
+
+module Scout
+
+  LOCK_DIRS = Scout.tmp.tsv_open_locks.find_all + Scout.tmp.persist_locks.find_all + Scout.tmp.sensiblewrite_locks.find_all +
+    Scout.tmp.produce_locks.find_all + Scout.tmp.step_info_locks.find_all
+
+  SENSIBLE_WRITE_DIRS = Open.sensible_write_dir.find_all
+
+  PERSIST_DIRS    = Scout.share.find_all  + Scout.var.cache.persistence.find_all
+
+  JOB_DIRS = Scout.var.jobs.find_all
+
+  MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
+
+  def self.dump_memory(file, obj = nil)
+    Log.info "Dumping #{obj} objects into #{ file }"
+    Thread.new do
+      while true
+        Open.write(file) do |f|
+          MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
+            GC.start
+            ObjectSpace.each_object(obj) do |o|
+              f.puts "---"
+              f.puts(String === o ? o : o.inspect)
+            end
+          end
+        end
+        FileUtils.cp file, file + '.save'
+        sleep 3
+      end
+    end
+  end
+
+  def self.file_time(file)
+    info = {}
+    begin
+      info[:ctime] = File.ctime file
+      info[:atime] = File.atime file
+      info[:elapsed] = Time.now - info[:ctime]
+    rescue Exception
+    end
+    info[:ctime] = Time.now - 999
+    info
+  end
+
+  #{{{ LOCKS
+
+  def self.locks(dirs = LOCK_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -name "*.lock" 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.lock_info(dirs = LOCK_DIRS)
+    lock_info = {}
+    locks(dirs).each do |f|
+      lock_info[f] = {}
+      begin
+        lock_info[f].merge!(file_time(f))
+        if File.size(f) > 0
+          info = Open.open(f) do |s|
+            Open.yaml(s)
+          end
+          IndiferentHash.setup(info)
+          lock_info[f][:pid] = info[:pid]
+          lock_info[f][:ppid] = info[:ppid]
+        end
+      rescue Exception
+        Log.exception $!
+      end
+    end
+    lock_info
+  end
+
+  #{{{ SENSIBLE WRITES
+
+  def self.sensiblewrites(dirs = SENSIBLE_WRITE_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -not -name "*.lock" -not -type d 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.sensiblewrite_info(dirs = SENSIBLE_WRITE_DIRS)
+    info = {}
+    sensiblewrites(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+
+  def self.persists(dirs = PERSIST_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -name "*.persist" 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.persist_info(dirs = PERSIST_DIRS)
+    info = {}
+    persists(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+
+  def self.job_info(workflows = nil, tasks = nil, dirs = JOB_DIRS)
+    require 'rbbt/workflow/step'
+
+    workflows = [workflows] if workflows and not Array === workflows
+    workflows = workflows.collect{|w| w.to_s} if workflows
+
+    tasks = [tasks] if tasks and not Array === tasks
+    tasks = tasks.collect{|w| w.to_s} if tasks
+
+    jobs = {}
+    seen = Set.new
+    _files = Set.new
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+
+      task_dir_workflows = {}
+      tasks_dirs = if dir == '.'
+                    ["."]
+                   else
+                     #workflowdirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                     #                 repo_dir, sub_path = dir_sub_path
+                     #                 Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                     #               else
+                     #                 dir.glob("*")
+                     #               end
+
+                     workflowdirs = dir.glob("*")
+
+                     workflowdirs.collect do |workflowdir|
+                       workflow = File.basename(workflowdir)
+                       next if workflows and not workflows.include? workflow
+
+                       #task_dirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                       #              repo_dir, sub_path = dir_sub_path
+                       #              Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                       #            else
+                       #              workflowdir.glob("*")
+                       #            end
+
+                       task_dirs = workflowdir.glob("*")
+
+                       task_dirs.each do |tasks_dir|
+                         task_dir_workflows[tasks_dir] = workflow
+                       end
+                     end.compact.flatten
+                   end
+
+      tasks_dirs.collect do |taskdir|
+        task = File.basename(taskdir)
+        next if tasks and not tasks.include? task
+
+
+        #files = if (dir_sub_path = Open.find_repo_dir(taskdir))
+        #          repo_dir, sub_path = dir_sub_path
+        #          Open.list_repo_files(*dir_sub_path).reject do |f|
+        #            f.include?("/.info/") ||
+        #              f.include?(".files/") ||
+        #              f.include?(".pid/") ||
+        #              File.directory?(f)
+        #          end.collect do |f|
+        #            File.join(repo_dir, f)
+        #          end
+        #        else
+        #          #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+        #          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+        #          CMD.cmd(cmd, :pipe => true).read.split("\n")
+        #        end
+
+        files = begin
+                  cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+                  CMD.cmd(cmd, :pipe => true).read.split("\n")
+                end
+
+        files = files.sort_by{|f| Open.mtime(f) || Time.now}
+        workflow = task_dir_workflows[taskdir]
+        TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
+          _files << file
+          if m = file.match(/(.*)\.(info|pid|files)$/)
+            file = m[1]
+          end
+          next if seen.include? file
+          seen << file
+
+          name = file[taskdir.length+1..-1]
+          info_file = file + '.info'
+
+          info = {}
+
+          info[:workflow] = workflow
+          info[:task] = task
+          info[:name] = name
+
+          if Open.exists? file
+            info = info.merge(file_time(file))
+            info[:done] = true
+            info[:info_file] = Open.exist?(info_file) ? info_file : nil
+          else
+            info = info.merge({:info_file => info_file, :done => false})
+          end
+
+          [file, info]
+        end
+
+      end.compact.flatten
+    end.compact.flatten
+    jobs
+  end
+
+  # REST
+
+  def self.__jobs(dirs = JOB_DIRS)
+    job_files = {}
+    dirs.each do |dir|
+      workflow_dirs = dir.glob("*").each do |wdir|
+        workflow = File.basename(wdir)
+        job_files[workflow] = {}
+        task_dirs = wdir.glob('*')
+        task_dirs.each do |tdir|
+          task = File.basename(tdir)
+          job_files[workflow][task] = tdir.glob('*')
+        end
+      end
+    end
+    jobs = {}
+    job_files.each do |workflow,task_jobs|
+      jobs[workflow] ||= {}
+      task_jobs.each do |task, files|
+        jobs[workflow][task] ||= {}
+        files.each do |f|
+          next if f =~ /\.lock$/
+          job = f.sub(/\.(info|files)/,'')
+
+          jobs[workflow][task][job] ||= {}
+          if jobs[workflow][task][job][:status].nil?
+            status = nil
+            status = :done if Open.exists? job
+            if status.nil? and f=~/\.info/
+              info = begin
+                       Step::INFO_SERIALIZER.load(Open.read(f, :mode => 'rb'))
+                     rescue
+                       {}
+                     end
+              status = info[:status]
+              pid = info[:pid]
+            end
+
+            jobs[workflow][task][job][:pid] = pid if pid
+            jobs[workflow][task][job][:status] = status if status
+          end
+        end
+      end
+    end
+    jobs
+  end
+
+  def self.load_lock(lock)
+    begin
+      info = Misc.insist 3 do
+        Open.yaml(lock)
+      end
+      info.values_at "pid", "ppid", "time"
+    rescue Exception
+      time = begin
+               File.atime(lock)
+             rescue Exception
+               Time.now
+             end
+      [nil, nil, time]
+    end
+  end
+
+end

data/lib/scout/workflow/deployment/orchestrator.rb

@@ -191,6 +191,7 @@ module Workflow
     def process(rules, jobs = nil)
       jobs, rules = rules, {} if jobs.nil?
       jobs = [jobs] if Step === jobs
+      failed_jobs = []
       begin
 
         workload = Orchestrator.workload(jobs)
@@ -199,7 +200,6 @@ module Workflow
         all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
 
         top_level_jobs = jobs.collect{|job| job.path }
-        failed_jobs = []
         while workload.any? 
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)

data/lib/scout/workflow/documentation.rb

@@ -2,7 +2,7 @@ module Workflow
   attr_accessor :title, :description
 
   def self.doc_parse_first_line(str)
-    if str.match(/^([^\n]*)\n\n(.*)/sm)
+    if str.match(/^([^\n]*)\n\n(.*)/smu)
       str.replace $2
       $1
     else

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.8.2 ruby lib
+# stub: scout-gear 10.8.4 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.8.2".freeze
+  s.version = "10.8.4".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-06-13"
+  s.date = "1980-01-02"
   s.description = "Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -51,6 +51,7 @@ Gem::Specification.new do |s|
     "lib/scout/knowledge_base/query.rb",
     "lib/scout/knowledge_base/registry.rb",
     "lib/scout/knowledge_base/traverse.rb",
+    "lib/scout/monitor.rb",
     "lib/scout/persist/engine.rb",
     "lib/scout/persist/engine/fix_width_table.rb",
     "lib/scout/persist/engine/packed_index.rb",
@@ -143,6 +144,7 @@ Gem::Specification.new do |s|
     "scout_commands/rbbt",
     "scout_commands/resource/produce",
     "scout_commands/resource/sync",
+    "scout_commands/system/clean",
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
@@ -250,7 +252,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.6".freeze
+  s.rubygems_version = "3.6.7".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4
@@ -259,6 +261,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<matrix>.freeze, [">= 0".freeze])
   s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<csv>.freeze, [">= 0".freeze])
   s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0".freeze])
 end
 

data/scout_commands/system/clean

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/monitor'
+
+$0 = "scout #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean orphaned files
+
+$ rbbt system clean <workflow> <task>
+
+Specify workflow '.' and no task to examine the jobs of the current directory (usefull for web-server cache).
+
+-a--all Apply to all jobs, not only uncompleted
+-o--older* Clean jobs not access in some time
+-f--force Remove locks and files regardless of been active
+-q--quick Quick check
+-d--dirty Clean dirty jobs
+-h--help Print this help
+EOF
+
+scout_usage and exit 0 if options[:help]
+
+workflow, task = ARGV
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+force = options.delete :force
+dirty = options.delete :dirty
+time = options.delete :older
+
+ENV["SCOUT_DEBUG_CLEAN"] = 'false'
+ENV["SCOUT_UPDATE"] = 'true' if dirty
+
+time = Misc.timespan time, 'd' if time
+
+puts Log.color(:magenta, "# System clean")
+
+locks = Scout.lock_info
+if locks.any?
+  puts
+  puts Log.color(:magenta, "Locks:")
+  locks.each do |file,info|
+    if force or (info[:pid] && ! Misc.pid_alive?(info[:pid]))
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+persists = Scout.persist_info
+if persists.any?
+  puts
+  puts Log.color(:magenta, "Persist:")
+  persists.each do |file,info|
+    if force or (info[:pid] and Misc.pid_alive? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+sensiblewrites = Scout.sensiblewrite_info
+if sensiblewrites.any?
+  puts
+  puts Log.color(:magenta, "Writes:")
+  sensiblewrites.each do |file,info|
+    if force or (info[:pid] and Misc.pid_alive? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
+    end
+  end
+end
+
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
+puts
+puts Log.color(:magenta, "# Workflow clean")
+puts
+
+if workflow === ['.']
+  jobs = Scout.job_info ["all"], task, ['.']
+else
+  jobs = Scout.job_info workflow, task
+end
+
+workflows = {}
+
+TSV.traverse jobs do |file,i|
+  if options[:quick] and i[:done]
+    status = 'done'
+  else
+    info = begin
+             Open.open(i[:info_file]) do |f|
+               Step.load_info(f)
+             end
+           rescue
+             {:status => :noinfo}
+           end
+
+    done = Open.exists?(file) && ! Open.broken_link?(file)
+    pid = info[:pid] || (Open.exists?(file + '.pid') && Open.read(file + '.pid')) unless done
+
+    status = info[:status].to_s
+    if status != "noinfo"
+      status = :missing if status == "done" and not done
+      status = :nopid if status != "done" and pid.nil?
+      status = :dead if status != "done" and pid and not Misc.pid_alive?(pid)
+      status = :sync if status != "done" and done
+    end
+
+    status = :dirty if info[:status].to_s == "done" and dirty and Workflow.load_step(file).dirty?
+    if info[:status] == :error
+      begin
+        exception = info[:exception][:class]
+        Kernel.const_get exception
+        status = :non_recoverable if exception.superclass === ScoutException
+      rescue
+      end
+    end
+
+    status = status.to_s
+  end
+
+  if time and Open.exists?(file)
+    old = Time.now - Open.atime(file)
+    if old > time
+      status = 'old'
+    end
+  end
+
+  if (force && status !~ /done/) or 
+      status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
+      (status == "noinfo" and not done) or
+      status == "" 
+
+    puts "  Removing #{ file } - #{status}" 
+    Step.clean(file) 
+  end
+end
+

data/scout_commands/workflow/task

@@ -115,7 +115,7 @@ else
   when "serial"
     job.run(true) 
   when "local"
-    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 3
+    refresh = Scout::Config.get :refresh, :deploy, :local, :orchestrator, default: 0.5
     orchestrator = Workflow::Orchestrator.new refresh.to_f, "cpus" => Misc.processors
     orchestrator.process({}, job)
   when "slurm"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.8.2
+  version: 10.8.4
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-06-13 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -65,6 +65,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: juwelier
   requirement: !ruby/object:Gem::Requirement
@@ -120,6 +134,7 @@ files:
 - lib/scout/knowledge_base/query.rb
 - lib/scout/knowledge_base/registry.rb
 - lib/scout/knowledge_base/traverse.rb
+- lib/scout/monitor.rb
 - lib/scout/persist/engine.rb
 - lib/scout/persist/engine/fix_width_table.rb
 - lib/scout/persist/engine/packed_index.rb
@@ -212,6 +227,7 @@ files:
 - scout_commands/rbbt
 - scout_commands/resource/produce
 - scout_commands/resource/sync
+- scout_commands/system/clean
 - scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/cmd
@@ -334,7 +350,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 3.6.7
 specification_version: 4
 summary: basic gear for scouts
 test_files: []