scout-gear 6.0.0 → 7.1.0

data/VERSION

@@ -1 +1 @@
-6.0.0
+7.1.0

data/lib/scout/exceptions.rb

@@ -84,6 +84,14 @@ class DoneProcessing < Exception
   end
 end
 
+class WorkerException < ScoutException
+  attr_accessor :exception, :pid
+  def initialize(exception, pid)
+    @exception = exception
+    @pid = pid
+  end
+end
+
 
 #class OpenGzipError < StandardError; end
 #

data/lib/scout/log/color.rb

@@ -143,13 +143,14 @@ module Log
   CONCEPT_COLORS = IndiferentHash.setup({
     :title => magenta,
     :path => blue,
-    :input => blue,
+    :input => cyan,
     :value => green,
     :integer => green,
     :negative => red,
     :float => green,
     :waiting => yellow,
-    :started => blue,
+    :started => cyan,
+    :start => cyan,
     :done => green,
     :error => red,
   })
@@ -166,10 +167,36 @@ module Log
   def self.color(color, str = nil, reset = false)
     return str.dup || "" if nocolor 
 
+    if (color == :integer || color == :float) && Numeric === str
+      color = if str < 0
+                :red
+              elsif str > 1
+                :cyan
+              else
+                :green
+              end
+    end
+
+    if color == :status 
+      color = case str.to_sym
+              when :done
+                :green
+              when :error, :aborted
+                :red
+              when :waiting, :queued
+                :yellow
+              when :started, :start, :streamming
+                :cyan
+              else
+                :cyan
+              end
+    end
+
     color = SEVERITY_COLOR[color] if Integer === color
     color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
     color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
 
+    str = str.to_s unless str.nil?
     return str if Symbol === color
     color_str = reset ? Term::ANSIColor.reset : ""
     color_str << color

data/lib/scout/log/progress/util.rb

@@ -28,6 +28,8 @@ module Log
     end
 
     def self.new_bar(max, options = {})
+      options, max = max, nil if Hash === max
+      max = options[:max] if max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"

data/lib/scout/log/progress.rb

@@ -24,6 +24,8 @@ module Log
         IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
         :depth => 0, :frequency => 2
 
+      max = nil if TrueClass === max
+
       @max = max
       @ticks = 0
       @frequency = frequency

data/lib/scout/log.rb

@@ -206,7 +206,11 @@ module Log
       line = line.sub('`',"'")
       color = :green if line =~ /workflow/
       color = :blue if line =~ /scout-/
-      Log.color color, line
+      if color
+        Log.color color, line
+      else
+        line
+      end
     end unless stack.nil?
   end
 

data/lib/scout/misc/digest.rb

@@ -41,9 +41,7 @@ module Misc
 
   def self.digest(obj)
     str = Misc.digest_str(obj)
-    hash = Digest::MD5.hexdigest(str)
-    Log.debug "Digest #{hash} - #{str}"
-    hash
+    Digest::MD5.hexdigest(str)
   end
 
   def self.file_md5(file)

data/lib/scout/open/stream.rb

@@ -31,13 +31,16 @@ module Open
         Thread.current.report_on_exception = false
         consume_stream(io, false, into, into_close)
       end
+
       io.threads.push(consumer_thread) if io.respond_to?(:threads)
+      Thread.pass until consumer_thread["name"]
+
       consumer_thread
     else
       if into
-        Log.medium "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
+        Log.low "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
       else
-        Log.medium "Consuming stream #{Log.fingerprint io}"
+        Log.low "Consuming stream #{Log.fingerprint io}"
       end
 
       begin
@@ -53,7 +56,6 @@ module Open
         into_close = false unless into.respond_to? :close
         io.sync = true
 
-        Log.high "started consuming stream #{Log.fingerprint io}"
         begin
           while c = io.readpartial(BLOCK_SIZE)
             into << c if into
@@ -67,15 +69,14 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        Log.high "Done consuming stream #{Log.fingerprint io} into #{into_path || into}"
         c
       rescue Aborted
-        Log.high "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
+        Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
-        Log.high "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
+        Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
         exception = io.stream_exception || $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
@@ -145,12 +146,12 @@ module Open
 
           Open.notify_write(path) 
         rescue Aborted
-          Log.medium "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.medium "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
@@ -219,16 +220,15 @@ module Open
 
       #parent_pid = Process.pid
       pid = Process.fork {
-        purge_pipes(sin)
-        sout.close
         begin
+          purge_pipes(sin)
+          sout.close
 
           yield sin
           sin.close if close and not sin.closed? 
 
         rescue Exception
           Log.exception $!
-          #Process.kill :INT, parent_pid
           Kernel.exit!(-1)
         end
         Kernel.exit! 0
@@ -242,18 +242,18 @@ module Open
       ConcurrentStream.setup sout, :pair => sin
 
       thread = Thread.new do 
-        Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-        Thread.current.report_on_exception = false
         begin
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
           
           yield sin
 
           sin.close if close and not sin.closed? and not sin.aborted?
         rescue Aborted
-          Log.medium "Aborted open_pipe: #{$!.message}"
+          Log.low "Aborted open_pipe: #{$!.message}"
           raise $!
         rescue Exception
-          Log.medium "Exception in open_pipe: #{$!.message}"
+          Log.low "Exception in open_pipe: #{$!.message}"
           begin
             sout.threads.delete(Thread.current)
             sout.pair = []
@@ -269,6 +269,7 @@ module Open
 
       sin.threads = [thread]
       sout.threads = [thread]
+      Thread.pass until thread["name"]
     end
 
     sout
@@ -287,8 +288,8 @@ module Open
 
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
-        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
         Thread.current.report_on_exception = false
+        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
         begin
@@ -317,7 +318,7 @@ module Open
         out_pipes.each do |sout|
           sout.abort if sout.respond_to? :abort
         end
-        Log.medium "Tee aborting #{Log.fingerprint stream}"
+        Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
       rescue Exception
         begin
@@ -332,7 +333,7 @@ module Open
           in_pipes.each do |sin|
             sin.close unless sin.closed?
           end
-          Log.medium "Tee exception #{Log.fingerprint stream}"
+          Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
           Log.exception $!
         ensure
@@ -348,7 +349,7 @@ module Open
     out_pipes.each do |sout|
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
     end
-    splitter_thread.wakeup until splitter_thread["name"]
+    Thread.pass until splitter_thread["name"]
 
     main_pipe = out_pipes.first
     main_pipe.autojoin = true

data/lib/scout/tsv/parser.rb

@@ -0,0 +1,144 @@
+module TSV
+  def self.cast_value(value, cast)
+    if Array === value
+      value.collect{|e| cast_value(e, cast) }
+    else
+      value.send(cast)
+    end
+  end
+
+  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil)
+    items = line.split(sep, -1)
+
+    if positions.nil? && key == 0
+      key = items.shift
+    elsif positions.nil? 
+      key = items.delete(key)
+    else 
+      key, items = items[key], items.values_at(*positions)
+    end
+
+    items = case type
+            when :list
+              items
+            when :single
+              items.first
+            when :flat
+              [items]
+            when :double
+              items.collect{|i| i.split(sep2, -1) }
+            end
+
+    key = key.partition(sep2).first if type == :double
+
+    if cast
+      items = cast_value(items, cast)
+    end
+
+    [key, items]
+  end
+
+  def self.parse_stream(stream, data: nil, merge: true, type: :list, fix: true, bar: false, first_line: nil, **kargs, &block)
+    begin
+      bar = Log::ProgressBar.new_bar(bar) if bar
+
+      data = {} if data.nil?
+      merge = false if type != :double
+      line = first_line || stream.gets
+      while line
+        begin
+          line.strip!
+          line = Misc.fixutf8(line) if fix
+          bar.tick if bar
+          key, items = parse_line(line, type: type, **kargs)
+
+          if block_given?
+            res = block.call(key, items)
+            data[key] = res unless res.nil?
+            next
+          end
+
+          if ! merge || ! data.include?(key)
+            data[key] = items
+          else
+            current = data[key]
+            if merge == :concat
+              items.each_with_index do |new,i|
+                next if new.empty?
+                current[i].concat(new)
+              end
+            else
+              merged = []
+              items.each_with_index do |new,i|
+                next if new.empty?
+                merged[i] = current[i] + new
+              end
+              data[key] = merged
+            end
+          end
+        ensure
+          line = stream.gets
+        end
+      end
+      data
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+    end
+  end
+
+  def self.parse_header(stream, fix: true, header_hash: '#', sep: "\n")
+    raise "Closed stream" if IO === stream && stream.closed?
+
+    options = {}
+    preamble = []
+
+    # Get line
+
+    #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+    line = stream.gets
+    return {} if line.nil?
+    line = Misc.fixutf8 line.chomp if fix
+
+    # Process options line
+    if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
+      options = IndiferentHash.string2hash m.captures.first.chomp
+      line = stream.gets
+      line = Misc.fixutf8 line.chomp if line && fix
+    end
+
+    # Determine separator
+    sep = options[:sep] if options[:sep]
+
+    # Process fields line
+    preamble << line if line
+    while line && (TrueClass === header_hash || (String === header_hash && line.start_with?(header_hash)))
+      fields = line.split(sep, -1)
+      key_field = fields.shift
+      key_field = key_field.sub(header_hash, '') if String === header_hash && ! header_hash.empty?
+
+      line = (header_hash != "" ?  stream.gets : nil)
+      line = Misc.fixutf8 line.chomp if line
+      preamble << line if line
+      break if TrueClass === header_hash || header_hash == ""
+    end
+
+    preamble = preamble[0..-3] * "\n"
+
+    line ||= stream.gets
+
+    first_line = line
+
+    [options, key_field, fields, first_line, preamble]
+  end
+
+  def self.parse(stream, **kwargs)
+    options, key_field, fields, first_line, preamble = parse_header(stream)
+
+    options.each do |option,value|
+      option = option.to_sym
+      kwargs[option] = value unless kwargs.include?(option)
+    end
+    data = parse_stream(stream, first_line: first_line, **kwargs)
+    TSV.setup data, :key_field => key_field, :fields => fields
+  end
+end

data/lib/scout/tsv.rb

@@ -0,0 +1,14 @@
+require_relative 'meta_extension'
+require_relative 'tsv/parser'
+
+module TSV
+  extend MetaExtension
+  extension_attr :key_field, :fields
+
+  def self.open(file, options = {})
+    Open.open(file) do |f|
+      TSV.parse(f,**options)
+    end
+  end
+end
+

data/lib/scout/work_queue/worker.rb

@@ -1,16 +1,18 @@
 class WorkQueue
   class Worker
     attr_accessor :pid, :ignore_ouput
-    def initialize
+    def initialize(ignore_ouput = false)
+      @ignore_output = ignore_ouput
     end
 
     def run
       @pid = Process.fork do
+        Log.debug "Worker start with #{Process.pid}"
         yield
       end
     end
 
-    def process(input, output, &block)
+    def process(input, output = nil, &block)
       run do
         begin
           while obj = input.read
@@ -19,33 +21,36 @@ class WorkQueue
               raise obj 
             end
             res = block.call obj
-            output.write res unless ignore_ouput || res == :ignore 
+            output.write res unless output.nil? || ignore_ouput || res == :ignore 
           end
         rescue DoneProcessing
-          Log.log "Worker #{Process.pid} done"
+        rescue Interrupt
         rescue Exception
-          Log.exception $!
+          output.write WorkerException.new($!, Process.pid)
           exit -1
         end
       end
     end
 
+    def abort
+      begin
+        Log.log "Aborting worker #{@pid}"
+        Process.kill "INT", @pid 
+      rescue Errno::ECHILD
+      end
+    end
+
     def join
       Log.log "Joining worker #{@pid}"
       Process.waitpid @pid
     end
 
-    def exit(status)
-      Log.log "Worker #{@pid} exited with status #{Log.color(:green, status)}"
-    end
-
     def self.join(workers)
       workers = [workers] unless Array === workers
       begin
         while pid = Process.wait 
           status = $?
-          worker = workers.select{|w| w.pid == pid }.first
-          worker.exit status.exitstatus if worker
+            worker = workers.select{|w| w.pid == pid }.first
         end
       rescue Errno::ECHILD
       end

data/lib/scout/work_queue.rb

@@ -35,52 +35,79 @@ class WorkQueue
   end
 
   def remove_worker(pid)
-    worker = @worker_mutex.synchronize do
-      Log.debug "Remove #{pid}"
-      @removed_workers.concat(@workers.delete_if{|w| w.pid == pid })
+    @worker_mutex.synchronize do
+      @workers.delete_if{|w| w.pid == pid }
+      @removed_workers << pid
     end
   end
 
   def process(&callback)
-    @workers.each do |w| 
-      w.process @input, @output, &@worker_proc
-    end
-    @reader = Thread.new do
+    @reader = Thread.new do |parent|
       begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Output reader #{Process.pid}"
+        @done_workers ||= []
         while true
           obj = @output.read
           if DoneProcessing === obj
-            remove_worker obj.pid if obj.pid
+            done = @worker_mutex.synchronize do
+              Log.low "Worker #{obj.pid} done"
+              @done_workers << obj.pid
+              @done_workers.length == @removed_workers.length + @workers.length
+            end
+            break if done
+          elsif Exception === obj
+            raise obj
           else
             callback.call obj if callback
           end
         end
+      rescue DoneProcessing
       rescue Aborted
+      rescue WorkerException
+        Log.error "Exception in worker #{obj.pid} #{Log.fingerprint obj.exception}"
+        self.abort
+        raise obj.exception
       end
-    end if @output
+    end
+
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    Thread.pass until @reader["name"]
+
+    @waiter = Thread.new do
+      begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Worker waiter #{Process.pid}"
+        while true
+          pid = Process.wait
+          remove_worker(pid)
+          break if workers.empty?
+        end
+      end
+    end
+
+    Thread.pass until @waiter["name"]
   end
 
   def write(obj)
     @input.write obj
   end
 
+  def abort
+    workers.each{|w| w.abort }
+  end
+
   def close
-    while @worker_mutex.synchronize{ @workers.length } > 0
-      begin
-        @input.write DoneProcessing.new
-        pid = Process.wait
-        status = $?
-        worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
-        worker.exit $?.exitstatus if worker
-      rescue Errno::ECHILD
-        Thread.pass until @workers.length == 0
-        break
-      end
+    @worker_mutex.synchronize{ @workers.length }.times do
+      @input.write DoneProcessing.new()
     end
-    @reader.raise Aborted if @reader
   end
 
   def join
+    @waiter.join if @waiter
     @reader.join if @reader
   end
 end

data/lib/scout/workflow/step/info.rb

@@ -55,9 +55,9 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path)
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
     else
-      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path) + " " + message
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
     end
   end
 

data/lib/scout/workflow/step.rb

@@ -109,7 +109,8 @@ class Step
   end
 
   def clean
-    FileUtils.rm path.find if path.exist?
+    Open.rm path if Open.exist?(path)
+    Open.rm info_file if Open.exist?(info_file)
   end
 
   def recursive_clean

data/lib/scout/workflow/task.rb

@@ -143,8 +143,8 @@ module Task
     non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     if non_default_inputs.any?
-      hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
-      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
+      hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
+      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
       id = [id, hash] * "_"
     end
 

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 6.0.0 ruby lib
+# stub: scout-gear 7.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "6.0.0"
+  s.version = "7.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-04-29"
+  s.date = "2023-05-01"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -77,6 +77,8 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/parse.rb",
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
+    "lib/scout/tsv.rb",
+    "lib/scout/tsv/parser.rb",
     "lib/scout/work_queue.rb",
     "lib/scout/work_queue/socket.rb",
     "lib/scout/work_queue/worker.rb",
@@ -137,8 +139,10 @@ Gem::Specification.new do |s|
     "test/scout/test_resource.rb",
     "test/scout/test_semaphore.rb",
     "test/scout/test_tmpfile.rb",
+    "test/scout/test_tsv.rb",
     "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
+    "test/scout/tsv/test_parser.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/step/test_info.rb",

data/test/scout/open/test_stream.rb

@@ -139,7 +139,7 @@ class TestOpenStream < Test::Unit::TestCase
 
   def test_tee_stream_save_error
     Log.with_severity 6 do
-      5.times do |i|
+      50.times do |i|
         TmpFile.with_file do |tmp|
           Path.setup tmp
           assert_raise ScoutException do

data/test/scout/test_semaphore.rb

@@ -4,7 +4,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 require 'scout/work_queue/worker'
 class TestSemaphore < Test::Unit::TestCase
 
-  def _test_simple
+  def test_simple
     ScoutSemaphore.with_semaphore 1 do |sem|
       10.times do
         ScoutSemaphore.synchronize(sem) do