scout-gear 10.11.4 → 10.11.7

data/lib/scout/tsv/dumper.rb

@@ -1,7 +1,7 @@
 module TSV
   class Dumper
     def self.header(options={})
-      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options, 
+      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options,
         :key_field, :fields, :sep, :header_hash, :preamble, :unnamed,
         :sep => "\t", :header_hash => "#", :preamble => true
 
@@ -29,8 +29,8 @@ module TSV
     attr_accessor :options, :initialized, :type, :sep, :compact, :filename, :namespace
     def initialize(options = {})
       options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
-      @sep, @type = IndiferentHash.process_options options, 
-        :sep, :type, 
+      @sep, @type = IndiferentHash.process_options options,
+        :sep, :type,
         :sep => "\t", :type => :double
       @compact = options[:compact]
       @options = options
@@ -54,7 +54,7 @@ module TSV
     def key_field
       @options[:key_field]
     end
-    
+
     def fields
       @options[:fields]
     end
@@ -62,7 +62,7 @@ module TSV
     def key_field=(key_field)
       @options[:key_field] = key_field
     end
-    
+
     def fields=(fields)
       @options[:fields] = fields
     end
@@ -100,7 +100,7 @@ module TSV
           when :double
             @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
           else
-            if Array === value 
+            if Array === value
               if Array === value.first
                 @sin << key + @sep + value.collect{|v| Array === v ? (@compact ? v.compact : v) * "|" : v } * @sep << "\n"
               else
@@ -147,7 +147,7 @@ module TSV
   end
 
   def dumper_stream(options = {})
-    preamble, unmerge, keys, stream = IndiferentHash.process_options options, 
+    preamble, unmerge, keys, stream = IndiferentHash.process_options options,
       :preamble, :unmerge, :keys, :stream,
       :preamble => true, :unmerge => false
     unmerge = false unless @type === :double
@@ -177,7 +177,7 @@ module TSV
 
     self.with_unnamed do
       if stream.nil?
-        t = Thread.new do 
+        t = Thread.new do
           begin
             Thread.current.report_on_exception = true
             Thread.current["name"] = "Dumper thread"

data/lib/scout/tsv/index.rb

@@ -61,7 +61,7 @@ module TSV
       if filename
         index = ScoutCabinet.open(filename, true, engine)
         TSV.setup(index, :type => :single)
-        index.extend TSVAdapter 
+        index.extend TSVAdapter
       else
         index = TSV.setup({}, :type => :single)
       end

data/lib/scout/tsv/open.rb

@@ -62,7 +62,7 @@ module Open
 
       if into.respond_to?(:close)
         obj = obj.find if Path === obj
-        into_thread = Thread.new do 
+        into_thread = Thread.new do
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Traverse into"
           error = false
@@ -96,7 +96,7 @@ module Open
       queue.process do |res|
         callback.call res if callback
       end
-      
+
       begin
         self.traverse(obj, **options) do |*args|
           queue.write args
@@ -210,7 +210,7 @@ module TSV
 
   def self.process_stream(stream, header_hash: "#", &block)
     sout = Open.open_pipe do |sin|
-      while line = stream.gets 
+      while line = stream.gets
         break unless line.start_with?(header_hash)
         sin.puts line
       end

data/lib/scout/tsv/stream.rb

@@ -77,7 +77,7 @@ module TSV
         fields = fields.compact.flatten
       end
 
-      options = input_options.first 
+      options = input_options.first
       type ||= options[:type]
       type ||= :list if type == :single
       type ||= :double if type == :flat
@@ -152,7 +152,7 @@ module TSV
 
                 if k == keys[i]
                   new_parts = NamedArray.zip_fields([new_parts]).zip(p).collect{|p| [p.flatten * "|"] }
-                  raise TryAgain 
+                  raise TryAgain
                 end
                 keys[i]= k
                 parts[i]= p

data/lib/scout/tsv/traverse.rb

@@ -60,9 +60,9 @@ module TSV
                 key = values.delete_at(key_pos)
               end
             end
-          else 
+          else
             orig_key = key
-            key = @type == :flat ? values : values[key_pos] if key_pos != :key 
+            key = @type == :flat ? values : values[key_pos] if key_pos != :key
 
             values = values.values_at(*positions)
             if key_index
@@ -85,7 +85,7 @@ module TSV
 
           values = TSV.cast_value(values, cast) if cast
 
-          if Array === key 
+          if Array === key
             key = key.uniq if uniq
             if @type == :double && one2one
               if one2one == :strict
@@ -159,7 +159,7 @@ module TSV
         end
       end
     end
-    
+
 
     [key_name, field_names]
   end

data/lib/scout/tsv/util/filter.rb

@@ -27,7 +27,7 @@ module Filtered
       @value = value
       @unsaved = []
 
-      case 
+      case
       when Hash === persistence
         @persistence = persistence
       when String === persistence
@@ -57,7 +57,7 @@ module Filtered
           self
         end.class_eval <<-EOC
           def match_entry(key, entry)
-            value = entry[@fieldnum] 
+            value = entry[@fieldnum]
             value == @value or (Array === value and value.include? @value)
           end
         EOC
@@ -67,7 +67,7 @@ module Filtered
     end
 
     def key
-      case 
+      case
       when String === value
         value
       else
@@ -162,7 +162,7 @@ module Filtered
     if not base.respond_to? :unfiltered_set
       class << base
         attr_accessor :filter_dir, :filters
-        
+
         alias unfiltered_set []=
         alias []= filtered_set
 
@@ -200,7 +200,7 @@ module Filtered
     if filters.empty?
       self.send(:unfiltered_set, key, value, clean)
     else
-      filters.each do |filter| 
+      filters.each do |filter|
         filter.add key if filter.match_entry key, value
       end
       self.send(:unfiltered_set, key, value, clean)
@@ -215,10 +215,10 @@ module Filtered
     end
   end
 
-  def filtered_values 
-    if filters.empty?  
-      self.send(:unfiltered_values) 
-    else 
+  def filtered_values
+    if filters.empty?
+      self.send(:unfiltered_values)
+    else
       ids = filters.inject(nil){|list,filter| list.nil? ? filter.ids.dup : Misc.intersect_sorted_arrays(list, filter.ids.dup)}
       self.send :values_at, *ids
     end

data/lib/scout/tsv/util/process.rb

@@ -13,7 +13,7 @@ module TSV
         field_values = values[field_pos]
       end
 
-      new_values = case 
+      new_values = case
                    when block.arity == 1
                      yield(field_values)
                    when block.arity == 2
@@ -54,7 +54,7 @@ module TSV
         values = new_values
       when (values.nil? and (fields.nil? or fields.empty?))
         values = [new_values]
-      when values.nil?  
+      when values.nil?
         values = [nil] * fields.length + [new_values]
       when Array === values
         values += [new_values]

data/lib/scout/tsv/util/reorder.rb

@@ -1,7 +1,7 @@
 require 'matrix'
 
 module TSV
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs) 
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, data: nil, unnamed: true, **kwargs)
     res = data || self.annotate({})
     res.type = kwargs[:type] if kwargs.include?(:type)
     kwargs[:one2one] = one2one
@@ -61,7 +61,7 @@ module TSV
     new = self.annotate({})
     TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
 
-    m = Matrix.rows values 
+    m = Matrix.rows values
     new_rows = m.transpose.to_a
 
     fields.zip(new_rows) do |key,row|

data/lib/scout/tsv/util/select.rb

@@ -41,7 +41,7 @@ module TSV
       if block_given?
         field = method
         field = fields.index?(field) if fields && String === field
-        case 
+        case
         when block.arity == 1
           if (method == key_field or method == :key)
             yield(key)
@@ -69,7 +69,7 @@ module TSV
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
     self.annotate(new)
-    
+
     case
     when (method.nil? and block_given?)
       through do |key, values|
@@ -101,7 +101,7 @@ module TSV
       end
     when ((String === method) || (Symbol === method))
       if block_given?
-        case 
+        case
         when block.arity == 1
           with_unnamed do
             case

data/lib/scout/tsv/util/unzip.rb

@@ -20,7 +20,7 @@ module TSV
              else
                target
              end
-               
+
     target.fields = new_fields
     target.key_field = new_key_field
     target.type = type
@@ -57,7 +57,7 @@ module TSV
             [new_key, new_values]
           end
         end
-        
+
         MultipleResult.setup(res)
       else
         field_value = v[field_pos]

data/lib/scout/tsv/util.rb

@@ -129,7 +129,7 @@ module TSV
 
     filename = @filename
     filename = "No filename" if filename.nil? || String === filename && filename.empty?
-    filename.find if Path === filename 
+    filename.find if Path === filename
     filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
 
     with_unnamed do

data/lib/scout/tsv.rb

@@ -68,7 +68,7 @@ module TSV
   end
 
   def self.str_setup(option_str, obj)
-    options = TSV.str2options(option_str) 
+    options = TSV.str2options(option_str)
     setup(obj, **options)
   end
 
@@ -87,7 +87,7 @@ module TSV
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
-    source_name, options = 
+    source_name, options =
       case file
       when StringIO
         [file.inspect, options]

data/lib/scout/work_queue/socket.rb

@@ -22,6 +22,7 @@ class WorkQueue
     end
 
     def clean
+      return if @cleaned
       @cleaned = true
       @sread.close unless @sread.closed?
       @swrite.close unless @swrite.closed?
@@ -56,9 +57,9 @@ class WorkQueue
       end
 
       write_length = str.length
-      wrote = stream.write(str) 
+      wrote = stream.write(str)
       while wrote < write_length
-        wrote += stream.write(str[wrote..-1]) 
+        wrote += stream.write(str[wrote..-1])
       end
     end
 

data/lib/scout/work_queue/worker.rb

@@ -41,10 +41,10 @@ class WorkQueue
           while obj = input.read
             if DoneProcessing === obj
               output.write DoneProcessing.new
-              raise obj 
+              raise obj
             end
             res = block.call obj
-            output.write res unless ignore_ouput || res == :ignore 
+            output.write res unless ignore_ouput || res == :ignore
           end
         rescue DoneProcessing
         rescue Interrupt
@@ -64,7 +64,7 @@ class WorkQueue
       begin
         Log.medium "Aborting worker #{worker_id}"
         Process.kill SIGNAL, @pid
-      rescue Errno::ECHILD 
+      rescue Errno::ECHILD
       rescue Errno::ESRCH
       end
     end
@@ -77,7 +77,7 @@ class WorkQueue
     def self.join(workers)
       workers = [workers] unless Array === workers
       begin
-        while pid = Process.wait 
+        while pid = Process.wait
           status = $?
             worker = workers.select{|w| w.pid == pid }.first
         end

data/lib/scout/work_queue.rb

@@ -62,7 +62,7 @@ class WorkQueue
   end
 
   def process(&callback)
-    @workers.each do |w| 
+    @workers.each do |w|
       w.process @input, @output, &@worker_proc
     end
 
@@ -126,10 +126,10 @@ class WorkQueue
           t
         end
         exceptions = []
-        threads.each do |t| 
+        threads.each do |t|
           begin
-            t.join 
-          rescue
+            t.join
+          rescue Exception
             exceptions << $!
           end
         end
@@ -155,10 +155,10 @@ class WorkQueue
     @aborted = true
     Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
     @worker_mutex.synchronize do
-      @workers.each do |w| 
+      @workers.each do |w|
         ScoutSemaphore.post_semaphore(@output.write_sem)
         ScoutSemaphore.post_semaphore(@input.read_sem)
-        w.abort 
+        w.abort
       end
     end
   end
@@ -175,7 +175,7 @@ class WorkQueue
   end
 
   def clean
-    @waiter.join if @waiter 
+    @waiter.join if @waiter && Thread.current != @waiter
     @input.clean
     @output.clean
   end

data/lib/scout/workflow/definition.rb

@@ -123,7 +123,7 @@ module Workflow
     block = lambda &self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
-      @annotate_next_task[:extension] ||=  
+      @annotate_next_task[:extension] ||=
         case type
         when :tsv
           "tsv"
@@ -144,24 +144,24 @@ module Workflow
     end
   end
 
-  FORGET_TASK_ALIAS = begin 
+  FORGET_TASK_ALIAS = begin
                         %w(SCOUT_FORGET_TASK_ALIAS SCOUT_FORGET_DEP_TASKS RBBT_FORGET_DEP_TASKS).select do |var|
                           ENV[var] == 'true'
                         end.any?
                       end
-  REMOVE_TASK_ALIAS = begin 
+  REMOVE_TASK_ALIAS = begin
                         remove = %w(SCOUT_REMOVE_TASK_ALIAS SCOUT_REMOVE_DEP_TASKS RBBT_REMOVE_DEP_TASKS).select do |var|
                           ENV.include?(var) && ENV[var] != 'false'
                         end.first
                         remove.nil? ? false : remove
                       end
   def task_alias(name, workflow, oname, *rest, &block)
-    dep(workflow, oname, *rest, &block) 
+    dep(workflow, oname, *rest, &block)
     extension :dep_task unless @extension
     task_proc = workflow.tasks[oname] if workflow.tasks
     if task_proc
       returns task_proc.returns if @returns.nil?
-      type = task_proc.type 
+      type = task_proc.type
     end
     task name => type do
       raise ScoutException, "dep_task does not have any dependencies" if dependencies.empty?
@@ -187,25 +187,27 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
 
-        if dep.overriden? 
+        if dep.overriden? || dep.overrider?
           Open.link dep.path, self.tmp_path
         else
           Open.ln_h dep.path, self.tmp_path
 
-          case remove.to_s
-          when 'true'
-            dep.clean
-          when 'recursive'
-            (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
-              next if d.overriden
-              d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
+          if Open.exists?(dep.info_file)
+            case remove.to_s
+            when 'true'
+              dep.clean
+            when 'recursive'
+              (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
+                next if d.overriden
+                d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
+              end
+              dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
             end
-            dep.clean unless Scout::Config.get(:remove_dep, "task:#{dep.task_signature}", "task:#{dep.task_name}", "workflow:#{dep.workflow.name}", :default => true).to_s == 'false'
-          end 
+          end
         end
       else
         if Open.exists?(dep.files_dir)
-          Open.rm_rf self.files_dir 
+          Open.rm_rf self.files_dir
           Open.link dep.files_dir, self.files_dir
         end
         if defined?(RemoteStep) && RemoteStep === dep