scout-gear 10.11.3 → 10.11.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59a4e58c16692963459b2f3993829a34a48bedae88a879e5b0a349acb279e135
-  data.tar.gz: 5473ffe02a923a32ea39d1c8b72e9841c25716382e13975d93f846b83943c43a
+  metadata.gz: 7fa8d9cac544d5ca1f697bd9664fed77ee89084e0d900aa237f2cbd0367a9453
+  data.tar.gz: fb87a9fb7e8562a9a5b5db2aae244f111612cc7ff1d8e1325714d99611add74e
 SHA512:
-  metadata.gz: f0937e1aedac6d478e5164a26802f342e759c71a62d6e6abf81b3621191c1b7c4ad72b9db1f5579c6b1478fff749b734b5fe9926774aa783a54c3130cc676e79
-  data.tar.gz: fe8ba7061d5d381097acab46ec9ad591526fb2623a9b8e7739b1afd20079f0df5fce7ca35418c781b416993afd9dcdc5d611c6db008c7f246a45fea37c150f42
+  metadata.gz: 67697db2c2f3fb425fb8a56298934856aa7cceb7b0b1c43b8bba8cba51f867e2d2685fbd962c596c7527506a09a9e93face9816844e1fb8708a828a144071d69
+  data.tar.gz: 1534d012e107a68f01a6a747e029dd9c7e1497c08d16398d4964f9c1661b2afe6692a3f984077e7e1a66c86719cdc1788d749ae20fd2eed8526402cce4a75e4e

data/.vimproject

@@ -193,6 +193,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   }
   workflow=workflow{
    task
+   example
    list
    info
    write_info

data/VERSION

@@ -1 +1 @@
-10.11.3
+10.11.6

data/lib/scout/semaphore.rb

@@ -25,7 +25,7 @@ if continue
 
       # Create a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int create_semaphore(char* name, int value){
+  int create_semaphore_c(char* name, int value){
     sem_t* sem;
     sem = sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
     if (sem == SEM_FAILED){
@@ -39,7 +39,7 @@ if continue
 
       # Unlink (remove) a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int delete_semaphore(char* name){
+  int delete_semaphore_c(char* name){
     int ret = sem_unlink(name);
     if (ret == -1) {
       return -errno;
@@ -50,7 +50,7 @@ if continue
 
       # Wait (sem_wait) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int wait_semaphore(char* name){
+  int wait_semaphore_c(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
@@ -76,7 +76,7 @@ if continue
 
       # Post (sem_post) on a named semaphore. Return 0 on success, -errno on error.
       builder.c_singleton <<-EOF
-  int post_semaphore(char* name){
+  int post_semaphore_c(char* name){
     sem_t* sem;
     sem = sem_open(name, 0);
     if (sem == SEM_FAILED){
@@ -113,6 +113,11 @@ if continue
       s
     end
 
+    def self.exists?(name)
+      file = File.join('/dev/shm', 'sem.' + name[1..-1])
+      Open.exists? file
+    end
+
     # Errno numeric lists
     RETRIABLE_ERRNOS = [
       Errno::ENOENT,
@@ -155,24 +160,26 @@ if continue
     end
 
     # Safe wrappers that raise SystemCallError on final failure
-    def self.safe_create_semaphore(name, value, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore(name, value) }
+    def self.create_semaphore(name, value, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.create_semaphore_c(name, value) }
+      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
       if ret < 0
         raise SystemCallError.new("create_semaphore(#{name}) failed", -ret)
       end
       ret
     end
 
-    def self.safe_delete_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore(name) }
+    def self.delete_semaphore(name, **opts)
+      ret = with_retry(**opts) { ScoutSemaphore.delete_semaphore_c(name) }
       if ret < 0
         raise SystemCallError.new("delete_semaphore(#{name}) failed", -ret)
       end
       ret
     end
 
-    def self.safe_wait_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore(name) }
+    def self.wait_semaphore(name, **opts)
+      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
+      ret = with_retry(**opts) { ScoutSemaphore.wait_semaphore_c(name) }
       if ret < 0
         err = -ret
         if err == Errno::EINTR.new.errno
@@ -184,8 +191,9 @@ if continue
       ret
     end
 
-    def self.safe_post_semaphore(name, **opts)
-      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore(name) }
+    def self.post_semaphore(name, **opts)
+      raise SystemCallError.new("Semaphore missing (#{name})") unless self.exists?(name)
+      ret = with_retry(**opts) { ScoutSemaphore.post_semaphore_c(name) }
       if ret < 0
         raise SystemCallError.new("post_semaphore(#{name}) failed", -ret)
       end
@@ -198,7 +206,7 @@ if continue
 
       # wait_semaphore returns 0 on success or -errno on error
       begin
-        ScoutSemaphore.safe_wait_semaphore(sem)
+        ScoutSemaphore.wait_semaphore(sem)
       rescue SemaphoreInterrupted
         raise
       rescue SystemCallError => e
@@ -210,10 +218,13 @@ if continue
         yield
       ensure
         begin
-          ScoutSemaphore.safe_post_semaphore(sem)
+          ScoutSemaphore.post_semaphore(sem)
         rescue SystemCallError => e
           # Log but don't raise from ensure
-          Log.warn "post_semaphore(#{sem}) failed in ensure: #{e.message}"
+          # Log.warn "post_semaphore(#{sem}) failed in ensure: #{e.message}"
+
+          # Actually, do raise
+          raise e
         end
       end
     end
@@ -229,7 +240,7 @@ if continue
       begin
         Log.debug "Creating semaphore (#{ size }): #{file}"
         begin
-          ScoutSemaphore.safe_create_semaphore(file, size)
+          ScoutSemaphore.create_semaphore(file, size)
         rescue SystemCallError => e
           Log.error "Failed to create semaphore #{file}: #{e.message}"
           raise
@@ -239,7 +250,7 @@ if continue
       ensure
         Log.debug "Removing semaphore #{ file }"
         begin
-          ScoutSemaphore.safe_delete_semaphore(file)
+          ScoutSemaphore.delete_semaphore(file)
         rescue SystemCallError => e
           Log.warn "delete_semaphore(#{file}) failed: #{e.message}"
         end

data/lib/scout/work_queue/socket.rb

@@ -22,6 +22,7 @@ class WorkQueue
     end
 
     def clean
+      return if @cleaned
       @cleaned = true
       @sread.close unless @sread.closed?
       @swrite.close unless @swrite.closed?

data/lib/scout/work_queue/worker.rb

@@ -51,8 +51,9 @@ class WorkQueue
         rescue Exception
           begin
             output.write WorkerException.new($!, Process.pid)
-          ensure
             exit EXIT_STATUS
+          rescue
+            exit -1
           end
         end
         exit 0

data/lib/scout/work_queue.rb

@@ -115,7 +115,7 @@ class WorkQueue
         break if @worker_mutex.synchronize{ @workers.empty? }
         threads = @workers.collect do |w|
           t = Thread.new do
-            Thread.report_on_exception = false
+            Thread.current.report_on_exception = false
             Thread.current["name"] = "Worker waiter #{queue_id} worker #{w.pid}"
             pid, status = Process.wait2 w.pid
             remove_worker(pid) if pid
@@ -129,7 +129,7 @@ class WorkQueue
         threads.each do |t| 
           begin
             t.join 
-          rescue
+          rescue Exception
             exceptions << $!
           end
         end
@@ -169,13 +169,13 @@ class WorkQueue
     @worker_mutex.synchronize{ @workers.length }.times do
       begin
         @input.write DoneProcessing.new() unless @input.closed_write?
-      rescue IOError
+      rescue IOError,Errno::ENOENT
       end
     end
   end
 
   def clean
-    @waiter.join if @waiter 
+    @waiter.join if @waiter && Thread.current != @waiter
     @input.clean
     @output.clean
   end

data/lib/scout/workflow/step.rb

@@ -202,9 +202,9 @@ class Step
         reset_info :status => :setup, :issued => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
           :task_name => task_name, :workflow => workflow.to_s,
-          :provided_inputs => Annotation.purge(provided_inputs),
+          :provided_inputs => IndiferentHash.serializable(provided_inputs),
           :non_default_inputs => non_default_inputs,
-          :inputs => Annotation.purge(inputs), :input_names => input_names, :type => type,
+          :inputs => IndiferentHash.serializable(inputs), :input_names => input_names, :type => type,
           :dependencies => (dependencies || []) .collect{|d| d.path }
 
         run_dependencies

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.11.3 ruby lib
+# stub: scout-gear 10.11.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.11.3".freeze
+  s.version = "10.11.6".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -169,6 +169,7 @@ Gem::Specification.new do |s|
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
+    "scout_commands/workflow/example",
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
     "scout_commands/workflow/list",

data/scout_commands/workflow/cmd

@@ -2,6 +2,7 @@
 
 require 'scout'
 
+exec = $0
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
@@ -76,7 +77,7 @@ begin
           CMD.cmd_log('ruby', wf.libdir["test_workflow.rb"].find)
         else
           Log.info "No bootstrap for #{ workflow }, running examples instead"
-          CMD.cmd_log("scout workflow example #{ workflow }")
+          CMD.cmd_log("#{exec} workflow example #{ workflow }")
           exit 0
         end
       end

data/scout_commands/workflow/example

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'scout'
+
+cmd = $previous_commands ?
+  "scout #{$previous_commands.any? ? "#{$previous_commands * ' '} " : ''}#{File.basename(__FILE__)}" :
+  $PROGRAM_NAME
+
+options = SOPT.setup <<~EOF
+Run examples
+
+$ #{cmd} <workflow> [<task>] [<name>] -- [<task_options>]
+
+Run workflow examples. You can define tasks and example names. You can also override task options
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+help = IndiferentHash.process_options options, :help
+
+filename, _ = ARGV
+
+
+raise MissingParameterException, :filename if filename.nil?
+
+def run_task(workflow, task, name)
+  Log.info "Running example #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+
+  example_dir = workflow.libdir.examples[task][name].find
+  Log.debug "Using #{example_dir}"
+
+  ARGV.replace([workflow.to_s, task, '--load_inputs', example_dir, '--jobname', name,'-pf'] + $saved_args)
+
+  path = nil
+  success = nil
+  TmpFile.with_file do |res|
+    Open.open(res, :mode => 'w') do |file|
+      @pid = Process.fork{
+        STDOUT.reopen res
+        load Rbbt.share.rbbt_commands.workflow.task.find
+      }
+      Signal.trap(:INT) do
+        begin
+          Process.kill "INT", @pid
+        ensure
+          Kernel.exit! -1
+        end
+      end
+      file.close
+      begin
+        p,s = Process.waitpid2 @pid
+        success = s.success?
+      rescue Errno::ECHILD
+        success = true
+      end
+    end
+    sleep 0.5
+    path = Open.read(res).strip.split("\n").last if File.exist? res
+  end
+  path = "NO RESULT" if path.nil? or path.empty?
+
+  if success
+    Log.info "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    return [path, true]
+  else
+    Log.info "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    return [path, false]
+  end
+end
+
+workflow = ARGV.shift
+raise ParameterException if workflow.nil?
+
+task = ARGV.shift
+task = nil if task == '--'
+
+name = ARGV.shift if task
+name = nil if name == '--'
+
+orig_name = name
+
+$saved_args = ARGV.dup
+
+$saved_args.shift if $saved_args.include? '--'
+
+workflow = Workflow.require_workflow workflow
+
+tasks = task ? [task] : workflow.libdir.examples.glob('*').collect{|file| File.basename file }
+
+task_result = {}
+TSV.traverse tasks do |task|
+  names = name ? [name] : workflow.libdir.examples[task].glob('*').collect{|file| File.basename file }
+  TSV.traverse names  do |name|
+    success = run_task workflow, task, name
+    task_result[[task, name]] = success
+  end
+end
+
+
+task_result.each do |code,res|
+  task, name = code
+  path, success = res
+  if success
+    if orig_name
+      puts Open.read(path)
+    else
+      STDERR.puts "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+      puts path
+    end
+  else
+    STDERR.puts "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    puts path
+  end
+end

data/scout_commands/workflow/info

@@ -38,6 +38,7 @@ elsif options[:recursive_inputs]
   end
 else
   step.info.each do |k,v|
+    next if k.to_s == 'exception'
     case v
     when nil
       next
@@ -50,5 +51,13 @@ else
   rescue
     next
   end
+  if step.error?
+    if exception = step.exception
+      begin
+        Log.exception exception
+      rescue
+      end
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.11.3
+  version: 10.11.6
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -252,6 +252,7 @@ files:
 - scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/cmd
+- scout_commands/workflow/example
 - scout_commands/workflow/info
 - scout_commands/workflow/install
 - scout_commands/workflow/list