scout-gear 10.11.2 → 10.11.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77727bffd885a022ae17b23477024af34da91584f2ac55cd410483d1b6074e5a
-  data.tar.gz: b6631c3b02c5c90194c60e68c0a6d6abd1b45ea74c3152422328488e8397f524
+  metadata.gz: f4ce69534bfbce4c385cfc9c23b280c6bb935a9f7228afbf7e585c3f093a5a18
+  data.tar.gz: 98dee8673df93baf527d75df6fb9358b9ec1ec54a37e60c38f0bcb34920fb124
 SHA512:
-  metadata.gz: 1ad9229647dfe17146a3e5d3eac31347dde32e5b74f5e866c42b24a1e82816a2a9ef9a3ee61f020e18479decdf0d1c0c7b8fb33afa742f9d3b88602b86dc6c04
-  data.tar.gz: f51b0bdb79d3aa42f27697ad01b598ee79e0eb71ecaa172f38005926c1a0c13b08751475ff7d374699b7d1f42922222c025a5b6cb2430864e69b064b63168009
+  metadata.gz: 81b897cea0e0d4153fd1d97cae601a17dc75c4c36ebb0c6fe0699a4440d3b7baddbb501fbdb48c409f89386379fcc153a67b7b53b71f95d460366c6993637986
+  data.tar.gz: d0cb23b7fa0b9b04e5719cab7ee365231105fb16e32f62aed37c22f886054de9538922b07cc4495371eb1356fbc9b98707ebfb23635dee847564623b53c50bce

data/.vimproject

@@ -3,6 +3,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  README.md
  chats=chats filter="*"{
 
+  semaphore
+
   deploy
 
 
@@ -202,6 +204,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    cmd
   }
   batch=batch{
+   tail
    list
    clean
   }

data/VERSION

@@ -1 +1 @@
-10.11.2
+10.11.4

data/lib/scout/semaphore.rb

@@ -23,70 +23,229 @@ if continue
   #include <fcntl.h>
       EOF
 
+      # Create a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void create_semaphore(char* name, int value){
-    sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+  int create_semaphore(char* name, int value){
+    sem_t* sem;
+    sem = sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+    if (sem == SEM_FAILED){
+      return -errno;
+    }
+    /* close our handle; the semaphore lives on until unlinked and all handles closed */
+    sem_close(sem);
+    return 0;
   }
       EOF
+
+      # Unlink (remove) a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void delete_semaphore(char* name){
-    sem_unlink(name);
+  int delete_semaphore(char* name){
+    int ret = sem_unlink(name);
+    if (ret == -1) {
+      return -errno;
+    }
+    return 0;
   }
       EOF
 
+      # Wait (sem_wait) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
   int wait_semaphore(char* name){
-    int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
-      return(errno);
+      return -errno;
     }
-    ret = sem_wait(sem);
+
+    int ret;
+    /* retry if interrupted by signal; stop on success or other error */
+    do {
+      ret = sem_wait(sem);
+    } while (ret == -1 && errno == EINTR);
+
     if (ret == -1){
-      return(errno);
+      int e = errno;
+      sem_close(sem);
+      return -e;
     }
+
     sem_close(sem);
-    return(ret);
+    return 0;
   }
       EOF
 
+      # Post (sem_post) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  void post_semaphore(char* name){
+  int post_semaphore(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
-    sem_post(sem);
+    if (sem == SEM_FAILED){
+      return -errno;
+    }
+
+    int ret;
+    /* retry post if interrupted */
+    do {
+      ret = sem_post(sem);
+    } while (ret == -1 && errno == EINTR);
+
+    if (ret == -1) {
+      int e = errno;
+      sem_close(sem);
+      return -e;
+    }
+
     sem_close(sem);
+    return 0;
   }
       EOF
 
     end
 
     SEM_MUTEX = Mutex.new
+
+    def self.ensure_semaphore_name(file)
+      # Ensure a valid POSIX named semaphore name: must start with '/'
+      s = file.to_s.dup
+      # strip leading slashes and replace other slashes with underscores, then prepend single '/'
+      s.gsub!(%r{^/+}, '')
+      s = '/' + s.gsub('/', '_')
+      s
+    end
+
+    # Errno numeric lists
+    RETRIABLE_ERRNOS = [
+      Errno::ENOENT,
+      Errno::EIDRM,
+      Errno::EAGAIN,
+      Errno::EMFILE,
+      Errno::ENFILE,
+      Errno::EINTR
+    ].map { |c| c.new.errno }
+
+    FATAL_ERRNOS = [
+      Errno::EINVAL,
+      Errno::EACCES
+    ].map { |c| c.new.errno }
+
+    # Generic retry wrapper with exponential backoff + jitter
+    def self.with_retry(max_attempts: 6, base_delay: 0.01, max_delay: 1.0, jitter: 0.5, retriable: RETRIABLE_ERRNOS)
+      attempts = 0
+      while true
+        attempts += 1
+        ret = yield
+        # caller expects 0 on success, negative errno on failure
+        return ret if ret >= 0
+
+        err = -ret
+        # don't retry if it's clearly fatal or not in retriable list
+        if FATAL_ERRNOS.include?(err) || attempts >= max_attempts || !retriable.include?(err)
+          return ret
+        end
+
+        # exponential backoff with jitter
+        base = base_delay * (2 ** (attempts - 1))
+        sleep_time = [base, max_delay].min
+        # add jitter in range [0, jitter * sleep_time)
+        sleep_time += rand * jitter * sleep_time
+
+        Log.warn "Semaphore operation failed (errno=#{err}), retrying in #{'%.3f' % sleep_time}s (attempt #{attempts}/#{max_attempts})"
+        sleep(sleep_time)
+      end
+    end
+
+    # Safe wrappers that raise SystemCallError on final failure
+    def self.safe_create_semaphore(name, value, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore(name, value) }
+      if ret < 0
+        raise SystemCallError.new("create_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
+    def self.safe_delete_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore(name) }
+      if ret < 0
+        raise SystemCallError.new("delete_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
+    def self.safe_wait_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore(name) }
+      if ret < 0
+        err = -ret
+        if err == Errno::EINTR.new.errno
+          raise SemaphoreInterrupted
+        else
+          raise SystemCallError.new("wait_semaphore(#{name}) failed", err)
+        end
+      end
+      ret
+    end
+
+    def self.safe_post_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore(name) }
+      if ret < 0
+        raise SystemCallError.new("post_semaphore(#{name}) failed", -ret)
+      end
+      ret
+    end
+
     def self.synchronize(sem)
-      ret = ScoutSemaphore.wait_semaphore(sem)
-      raise SemaphoreInterrupted if ret == -1
+      # Ensure name is normalized (caller should pass normalized name, but be safe)
+      sem = ensure_semaphore_name(sem)
+
+      # wait_semaphore returns 0 on success or -errno on error
+      begin
+        ScoutSemaphore.safe_wait_semaphore(sem)
+      rescue SemaphoreInterrupted
+        raise
+      rescue SystemCallError => e
+        # bubble up for callers to handle
+        raise
+      end
+
       begin
         yield
       ensure
-        ScoutSemaphore.post_semaphore(sem)
+        begin
+          ScoutSemaphore.safe_post_semaphore(sem)
+        rescue SystemCallError => e
+          # Log but don't raise from ensure
+          # Log.warn "post_semaphore(#{sem}) failed in ensure: #{e.message}"
+
+          # Actually, do raise
+          raise e
+        end
       end
     end
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
+        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10]
       else
-        file = file.gsub('/', '_') if file
+        # ensure valid POSIX name
+        file = ensure_semaphore_name(file)
       end
 
       begin
         Log.debug "Creating semaphore (#{ size }): #{file}"
-        ScoutSemaphore.create_semaphore(file, size)
+        begin
+          ScoutSemaphore.safe_create_semaphore(file, size)
+        rescue SystemCallError => e
+          Log.error "Failed to create semaphore #{file}: #{e.message}"
+          raise
+        end
+
         yield file
       ensure
         Log.debug "Removing semaphore #{ file }"
-        ScoutSemaphore.delete_semaphore(file)
+        begin
+          ScoutSemaphore.safe_delete_semaphore(file)
+        rescue SystemCallError => e
+          Log.warn "delete_semaphore(#{file}) failed: #{e.message}"
+        end
       end
     end
 
@@ -114,16 +273,17 @@ if continue
 
         threads = []
         wait_mutex.synchronize do
-          threads = elems.collect do |elem| 
+          threads = elems.collect do |elem|
             Thread.new(elem) do |elem|
 
               continue = false
               mutex.synchronize do
                 while not continue do
-                  if count < size 
+                  if count < size
                     continue = true
                     count += 1
                   end
+                  # wait briefly to avoid busy loop; ConditionVariable could be used here properly
                   mutex.sleep 1 unless continue
                 end
               end
@@ -143,8 +303,8 @@ if continue
           end
         end
 
-        threads.each do |thread| 
-          thread.join 
+        threads.each do |thread|
+          thread.join
         end
       rescue Exception
         Log.exception $!
@@ -152,6 +312,5 @@ if continue
         threads.each do |thread| thread.kill end
       end
     end
-  end 
+  end
 end
-

data/lib/scout/work_queue/worker.rb

@@ -51,8 +51,9 @@ class WorkQueue
         rescue Exception
           begin
             output.write WorkerException.new($!, Process.pid)
-          ensure
             exit EXIT_STATUS
+          rescue
+            exit -1
           end
         end
         exit 0

data/lib/scout/work_queue.rb

@@ -71,9 +71,8 @@ class WorkQueue
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{queue_id}"
         @done_workers ||= []
-        #while true
-        #  obj = @output.read
-        while obj = @output.read
+        while true
+          obj = @output.read
           if DoneProcessing === obj
 
             done = @worker_mutex.synchronize do
@@ -90,6 +89,7 @@ class WorkQueue
             callback.call obj if callback
           end
         end
+        @waiter.join if @workers.any?
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
@@ -115,7 +115,7 @@ class WorkQueue
         break if @worker_mutex.synchronize{ @workers.empty? }
         threads = @workers.collect do |w|
           t = Thread.new do
-            Thread.report_on_exception = false
+            Thread.current.report_on_exception = false
             Thread.current["name"] = "Worker waiter #{queue_id} worker #{w.pid}"
             pid, status = Process.wait2 w.pid
             remove_worker(pid) if pid
@@ -135,9 +135,6 @@ class WorkQueue
         end
 
         raise exceptions.first if exceptions.any?
-        if @workers.empty? && ! @closed
-          @output.write DoneProcessing.new
-        end
       end
     end
 
@@ -172,7 +169,7 @@ class WorkQueue
     @worker_mutex.synchronize{ @workers.length }.times do
       begin
         @input.write DoneProcessing.new() unless @input.closed_write?
-      rescue IOError
+      rescue IOError,Errno::ENOENT
       end
     end
   end

data/lib/scout/workflow/deployment/scheduler/job.rb

@@ -695,8 +695,17 @@ env > #{batch_options[:fenv]}
         out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exist?(fout) and not tail == :STDERR
         err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exist?(ferr)
 
-        terr = Misc.consume_stream(err, true, STDERR) if err
-        tout = Misc.consume_stream(out, true, STDOUT) if out
+        tout = Thread.new do
+          while c = out.getc
+            STDOUT << c
+          end
+        end
+
+        terr = Thread.new do
+          while c = err.getc
+            STDERR << c
+          end
+        end
 
         sleep 3 while job_queued(job)
       rescue Aborted

data/lib/scout/workflow/step/info.rb

@@ -206,7 +206,7 @@ class Step
       Marshal.load(Base64.decode64(info[:exception]))
     rescue
       Log.exception $!
-      nil
+      return Exception.new messages.last
     end
   end
 

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.11.2 ruby lib
+# stub: scout-gear 10.11.4 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.11.2".freeze
+  s.version = "10.11.4".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -147,6 +147,7 @@ Gem::Specification.new do |s|
     "scout_commands/alias",
     "scout_commands/batch/clean",
     "scout_commands/batch/list",
+    "scout_commands/batch/tail",
     "scout_commands/cat",
     "scout_commands/doc",
     "scout_commands/entity",
@@ -282,7 +283,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.7.2".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/batch/list

@@ -189,7 +189,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
   count += 1
 
-  if options[:compressed] 
+  if options[:compressed] && tail.nil?
     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
     if different_system
       status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  

data/scout_commands/batch/tail

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory|jobid> [options] 
+
+-h--help Print this help
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/scout-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
+
+require 'rbbt/hpc/slurm'
+
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Directory: ") + directory if directory
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
+SLURM.follow_job directory, true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.11.2
+  version: 10.11.4
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -230,6 +230,7 @@ files:
 - scout_commands/alias
 - scout_commands/batch/clean
 - scout_commands/batch/list
+- scout_commands/batch/tail
 - scout_commands/cat
 - scout_commands/doc
 - scout_commands/entity
@@ -380,7 +381,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: basic gear for scouts
 test_files: []