scout-gear 10.11.1 → 10.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aca5b47a57bc63063f01af5278f878b9cf6205f16c7bf7154db2c8ba30043627
-  data.tar.gz: b9df56fe4d812cba82bb2d30a7474a64867e99a6181501d8136da3861aaf72dd
+  metadata.gz: 59a4e58c16692963459b2f3993829a34a48bedae88a879e5b0a349acb279e135
+  data.tar.gz: 5473ffe02a923a32ea39d1c8b72e9841c25716382e13975d93f846b83943c43a
 SHA512:
-  metadata.gz: 0bfd403c3ecbe1750e5937c27db75b58d92e6bcccdb96b1470cbfb20e1c8c0cbe3fa14d2eef1dd2d8780a0edede5a600cd4968144c939be9074fcbf2c352ea27
-  data.tar.gz: ab126a4dfe118ec5083f2121909c6b0b51a0bddc617b9678908472a23eeb6dcc81c23e37a6e75d82a22381a096339e6fd92f6018501589141aaa650b8991f1ab
+  metadata.gz: f0937e1aedac6d478e5164a26802f342e759c71a62d6e6abf81b3621191c1b7c4ad72b9db1f5579c6b1478fff749b734b5fe9926774aa783a54c3130cc676e79
+  data.tar.gz: fe8ba7061d5d381097acab46ec9ad591526fb2623a9b8e7739b1afd20079f0df5fce7ca35418c781b416993afd9dcdc5d611c6db008c7f246a45fea37c150f42

data/.vimproject

@@ -3,6 +3,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  README.md
  chats=chats filter="*"{
 
+  semaphore
+
   deploy
 
 
@@ -202,6 +204,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    cmd
   }
   batch=batch{
+   tail
    list
    clean
   }

data/VERSION

@@ -1 +1 @@
-10.11.1
+10.11.3

data/lib/scout/semaphore.rb

@@ -23,70 +23,226 @@ if continue
   #include <fcntl.h>
       EOF
 
+      # Create a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void create_semaphore(char* name, int value){
-    sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+  int create_semaphore(char* name, int value){
+    sem_t* sem;
+    sem = sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+    if (sem == SEM_FAILED){
+      return -errno;
+    }
+    /* close our handle; the semaphore lives on until unlinked and all handles closed */
+    sem_close(sem);
+    return 0;
   }
       EOF
+
+      # Unlink (remove) a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void delete_semaphore(char* name){
-    sem_unlink(name);
+  int delete_semaphore(char* name){
+    int ret = sem_unlink(name);
+    if (ret == -1) {
+      return -errno;
+    }
+    return 0;
   }
       EOF
 
+      # Wait (sem_wait) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
   int wait_semaphore(char* name){
-    int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
-      return(errno);
+      return -errno;
     }
-    ret = sem_wait(sem);
+
+    int ret;
+    /* retry if interrupted by signal; stop on success or other error */
+    do {
+      ret = sem_wait(sem);
+    } while (ret == -1 && errno == EINTR);
+
     if (ret == -1){
-      return(errno);
+      int e = errno;
+      sem_close(sem);
+      return -e;
     }
+
     sem_close(sem);
-    return(ret);
+    return 0;
   }
       EOF
 
+      # Post (sem_post) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void post_semaphore(char* name){
+  int post_semaphore(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
-    sem_post(sem);
+    if (sem == SEM_FAILED){
+      return -errno;
+    }
+
+    int ret;
+    /* retry post if interrupted */
+    do {
+      ret = sem_post(sem);
+    } while (ret == -1 && errno == EINTR);
+
+    if (ret == -1) {
+      int e = errno;
+      sem_close(sem);
+      return -e;
+    }
+
     sem_close(sem);
+    return 0;
   }
       EOF
 
     end
 
     SEM_MUTEX = Mutex.new
+
+    def self.ensure_semaphore_name(file)
+      # Ensure a valid POSIX named semaphore name: must start with '/'
+      s = file.to_s.dup
+      # strip leading slashes and replace other slashes with underscores, then prepend single '/'
+      s.gsub!(%r{^/+}, '')
+      s = '/' + s.gsub('/', '_')
+      s
+    end
+
+    # Errno numeric lists
+    RETRIABLE_ERRNOS = [
+      Errno::ENOENT,
+      Errno::EIDRM,
+      Errno::EAGAIN,
+      Errno::EMFILE,
+      Errno::ENFILE,
+      Errno::EINTR
+    ].map { |c| c.new.errno }
+
+    FATAL_ERRNOS = [
+      Errno::EINVAL,
+      Errno::EACCES
+    ].map { |c| c.new.errno }
+
+    # Generic retry wrapper with exponential backoff + jitter
+    def self.with_retry(max_attempts: 6, base_delay: 0.01, max_delay: 1.0, jitter: 0.5, retriable: RETRIABLE_ERRNOS)
+      attempts = 0
+      while true
+        attempts += 1
+        ret = yield
+        # caller expects 0 on success, negative errno on failure
+        return ret if ret >= 0
+
+        err = -ret
+        # don't retry if it's clearly fatal or not in retriable list
+        if FATAL_ERRNOS.include?(err) || attempts >= max_attempts || !retriable.include?(err)
+          return ret
+        end
+
+        # exponential backoff with jitter
+        base = base_delay * (2 ** (attempts - 1))
+        sleep_time = [base, max_delay].min
+        # add jitter in range [0, jitter * sleep_time)
+        sleep_time += rand * jitter * sleep_time
+
+        Log.warn "Semaphore operation failed (errno=#{err}), retrying in #{'%.3f' % sleep_time}s (attempt #{attempts}/#{max_attempts})"
+        sleep(sleep_time)
+      end
+    end
+
+    # Safe wrappers that raise SystemCallError on final failure
+    def self.safe_create_semaphore(name, value, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore(name, value) }
+      if ret < 0
+        raise SystemCallError.new("create_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
+    def self.safe_delete_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore(name) }
+      if ret < 0
+        raise SystemCallError.new("delete_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
+    def self.safe_wait_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore(name) }
+      if ret < 0
+        err = -ret
+        if err == Errno::EINTR.new.errno
+          raise SemaphoreInterrupted
+        else
+          raise SystemCallError.new("wait_semaphore(#{name}) failed", err)
+        end
+      end
+      ret
+    end
+
+    def self.safe_post_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore(name) }
+      if ret < 0
+        raise SystemCallError.new("post_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
     def self.synchronize(sem)
-      ret = ScoutSemaphore.wait_semaphore(sem)
-      raise SemaphoreInterrupted if ret == -1
+      # Ensure name is normalized (caller should pass normalized name, but be safe)
+      sem = ensure_semaphore_name(sem)
+
+      # wait_semaphore returns 0 on success or -errno on error
+      begin
+        ScoutSemaphore.safe_wait_semaphore(sem)
+      rescue SemaphoreInterrupted
+        raise
+      rescue SystemCallError => e
+        # bubble up for callers to handle
+        raise
+      end
+
       begin
         yield
       ensure
-        ScoutSemaphore.post_semaphore(sem)
+        begin
+          ScoutSemaphore.safe_post_semaphore(sem)
+        rescue SystemCallError => e
+          # Log but don't raise from ensure
+          Log.warn "post_semaphore(#{sem}) failed in ensure: #{e.message}"
+        end
       end
     end
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
+        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10]
       else
-        file = file.gsub('/', '_') if file
+        # ensure valid POSIX name
+        file = ensure_semaphore_name(file)
       end
 
       begin
         Log.debug "Creating semaphore (#{ size }): #{file}"
-        ScoutSemaphore.create_semaphore(file, size)
+        begin
+          ScoutSemaphore.safe_create_semaphore(file, size)
+        rescue SystemCallError => e
+          Log.error "Failed to create semaphore #{file}: #{e.message}"
+          raise
+        end
+
         yield file
       ensure
         Log.debug "Removing semaphore #{ file }"
-        ScoutSemaphore.delete_semaphore(file)
+        begin
+          ScoutSemaphore.safe_delete_semaphore(file)
+        rescue SystemCallError => e
+          Log.warn "delete_semaphore(#{file}) failed: #{e.message}"
+        end
       end
     end
 
@@ -114,16 +270,17 @@ if continue
 
         threads = []
         wait_mutex.synchronize do
-          threads = elems.collect do |elem| 
+          threads = elems.collect do |elem|
             Thread.new(elem) do |elem|
 
               continue = false
               mutex.synchronize do
                 while not continue do
-                  if count < size 
+                  if count < size
                     continue = true
                     count += 1
                   end
+                  # wait briefly to avoid busy loop; ConditionVariable could be used here properly
                   mutex.sleep 1 unless continue
                 end
               end
@@ -143,8 +300,8 @@ if continue
           end
         end
 
-        threads.each do |thread| 
-          thread.join 
+        threads.each do |thread|
+          thread.join
         end
       rescue Exception
         Log.exception $!
@@ -152,6 +309,5 @@ if continue
         threads.each do |thread| thread.kill end
       end
     end
-  end 
+  end
 end
-

data/lib/scout/work_queue.rb

@@ -71,9 +71,8 @@ class WorkQueue
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{queue_id}"
         @done_workers ||= []
-        #while true
-        #  obj = @output.read
-        while obj = @output.read
+        while true
+          obj = @output.read
           if DoneProcessing === obj
 
             done = @worker_mutex.synchronize do
@@ -90,6 +89,7 @@ class WorkQueue
             callback.call obj if callback
           end
         end
+        @waiter.join if @workers.any?
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
@@ -135,9 +135,6 @@ class WorkQueue
         end
 
         raise exceptions.first if exceptions.any?
-        if @workers.empty? && ! @closed
-          @output.write DoneProcessing.new
-        end
       end
     end
 

data/lib/scout/workflow/deployment/local.rb

@@ -45,6 +45,7 @@ class Workflow::LocalExecutor
   end
 
   def process_batches(batches)
+    retry_jobs = []
     failed_jobs = []
 
     while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
@@ -54,6 +55,16 @@ class Workflow::LocalExecutor
 
       raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
 
+      if candidates.reject{|batch| failed_jobs.include? batch[:top_level] }.empty? && resources_used.empty? && top_level_jobs.empty?
+        exception = failed_jobs.collect(&:get_exception).compact.first
+        if exception
+          Log.warn 'Some work failed'
+          raise exception 
+        else
+          raise 'Some work failed'
+        end
+      end
+
       candidates.each do |batch|
         begin
 
@@ -63,15 +74,18 @@ class Workflow::LocalExecutor
           when (job.error? || job.aborted?)
             begin
               if job.recoverable_error?
-                if failed_jobs.include?(job)
+                if retry_jobs.include?(job)
                   Log.warn "Failed twice #{job.path} with recoverable error"
+                  retry_jobs.delete job
+                  failed_jobs << job
                   next
                 else
-                  failed_jobs << job
+                  retry_jobs << job
                   job.clean
                   raise TryAgain
                 end
               else
+                failed_jobs << job
                 Log.warn "Non-recoverable error in #{job.path}"
                 next
               end

data/lib/scout/workflow/deployment/scheduler/job.rb

@@ -695,8 +695,17 @@ env > #{batch_options[:fenv]}
         out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exist?(fout) and not tail == :STDERR
         err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exist?(ferr)
 
-        terr = Misc.consume_stream(err, true, STDERR) if err
-        tout = Misc.consume_stream(out, true, STDOUT) if out
+        tout = Thread.new do
+          while c = out.getc
+            STDOUT << c
+          end
+        end
+
+        terr = Thread.new do
+          while c = err.getc
+            STDERR << c
+          end
+        end
 
         sleep 3 while job_queued(job)
       rescue Aborted

data/lib/scout/workflow/step/info.rb

@@ -206,7 +206,7 @@ class Step
       Marshal.load(Base64.decode64(info[:exception]))
     rescue
       Log.exception $!
-      nil
+      return Exception.new messages.last
     end
   end
 

data/lib/scout/workflow/step/load.rb

@@ -19,6 +19,6 @@ class Step
   end
 
   def to_json(...)
-    self.path
+    self.path.to_json
   end
 end

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.11.1 ruby lib
+# stub: scout-gear 10.11.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.11.1".freeze
+  s.version = "10.11.3".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -147,6 +147,7 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/batch/tail",
     "scout_commands/cat",
     "scout_commands/doc",
     "scout_commands/entity",
@@ -282,7 +283,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.7.2".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/batch/list

@@ -189,7 +189,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
   count += 1
 
-  if options[:compressed] 
+  if options[:compressed] && tail.nil?
     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
     if different_system
       status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  

data/scout_commands/batch/tail

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory|jobid> [options] 
+
+-h--help Print this help
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/scout-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
+
+require 'rbbt/hpc/slurm'
+
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Directory: ") + directory if directory
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
+SLURM.follow_job directory, true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.11.1
+  version: 10.11.3
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -230,6 +230,7 @@ files:
 - scout_commands/alias
 - scout_commands/batch/clean
 - scout_commands/batch/list
+- scout_commands/batch/tail
 - scout_commands/cat
 - scout_commands/doc
 - scout_commands/entity
@@ -380,7 +381,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: basic gear for scouts
 test_files: []