scout-gear 5.2.0 → 7.1.0

data/lib/scout/tsv/parser.rb

@@ -0,0 +1,144 @@
+module TSV
+  def self.cast_value(value, cast)
+    if Array === value
+      value.collect{|e| cast_value(e, cast) }
+    else
+      value.send(cast)
+    end
+  end
+
+  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil)
+    items = line.split(sep, -1)
+
+    if positions.nil? && key == 0
+      key = items.shift
+    elsif positions.nil? 
+      key = items.delete(key)
+    else 
+      key, items = items[key], items.values_at(*positions)
+    end
+
+    items = case type
+            when :list
+              items
+            when :single
+              items.first
+            when :flat
+              [items]
+            when :double
+              items.collect{|i| i.split(sep2, -1) }
+            end
+
+    key = key.partition(sep2).first if type == :double
+
+    if cast
+      items = cast_value(items, cast)
+    end
+
+    [key, items]
+  end
+
+  def self.parse_stream(stream, data: nil, merge: true, type: :list, fix: true, bar: false, first_line: nil, **kargs, &block)
+    begin
+      bar = Log::ProgressBar.new_bar(bar) if bar
+
+      data = {} if data.nil?
+      merge = false if type != :double
+      line = first_line || stream.gets
+      while line
+        begin
+          line.strip!
+          line = Misc.fixutf8(line) if fix
+          bar.tick if bar
+          key, items = parse_line(line, type: type, **kargs)
+
+          if block_given?
+            res = block.call(key, items)
+            data[key] = res unless res.nil?
+            next
+          end
+
+          if ! merge || ! data.include?(key)
+            data[key] = items
+          else
+            current = data[key]
+            if merge == :concat
+              items.each_with_index do |new,i|
+                next if new.empty?
+                current[i].concat(new)
+              end
+            else
+              merged = []
+              items.each_with_index do |new,i|
+                next if new.empty?
+                merged[i] = current[i] + new
+              end
+              data[key] = merged
+            end
+          end
+        ensure
+          line = stream.gets
+        end
+      end
+      data
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+    end
+  end
+
+  def self.parse_header(stream, fix: true, header_hash: '#', sep: "\n")
+    raise "Closed stream" if IO === stream && stream.closed?
+
+    options = {}
+    preamble = []
+
+    # Get line
+
+    #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+    line = stream.gets
+    return {} if line.nil?
+    line = Misc.fixutf8 line.chomp if fix
+
+    # Process options line
+    if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
+      options = IndiferentHash.string2hash m.captures.first.chomp
+      line = stream.gets
+      line = Misc.fixutf8 line.chomp if line && fix
+    end
+
+    # Determine separator
+    sep = options[:sep] if options[:sep]
+
+    # Process fields line
+    preamble << line if line
+    while line && (TrueClass === header_hash || (String === header_hash && line.start_with?(header_hash)))
+      fields = line.split(sep, -1)
+      key_field = fields.shift
+      key_field = key_field.sub(header_hash, '') if String === header_hash && ! header_hash.empty?
+
+      line = (header_hash != "" ?  stream.gets : nil)
+      line = Misc.fixutf8 line.chomp if line
+      preamble << line if line
+      break if TrueClass === header_hash || header_hash == ""
+    end
+
+    preamble = preamble[0..-3] * "\n"
+
+    line ||= stream.gets
+
+    first_line = line
+
+    [options, key_field, fields, first_line, preamble]
+  end
+
+  def self.parse(stream, **kwargs)
+    options, key_field, fields, first_line, preamble = parse_header(stream)
+
+    options.each do |option,value|
+      option = option.to_sym
+      kwargs[option] = value unless kwargs.include?(option)
+    end
+    data = parse_stream(stream, first_line: first_line, **kwargs)
+    TSV.setup data, :key_field => key_field, :fields => fields
+  end
+end

data/lib/scout/tsv.rb

@@ -0,0 +1,14 @@
+require_relative 'meta_extension'
+require_relative 'tsv/parser'
+
+module TSV
+  extend MetaExtension
+  extension_attr :key_field, :fields
+
+  def self.open(file, options = {})
+    Open.open(file) do |f|
+      TSV.parse(f,**options)
+    end
+  end
+end
+

data/lib/scout/work_queue/socket.rb

@@ -0,0 +1,119 @@
+require 'scout/open'
+require 'scout/semaphore'
+require 'scout/exceptions'
+class WorkQueue
+  class Socket
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    def initialize(serializer = nil)
+      @sread, @swrite = Open.pipe
+
+      @serializer = serializer || Marshal
+
+      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @write_sem = @key + '.in'
+      @read_sem = @key + '.out'
+      Log.debug "Creating socket semaphores: #{@key}"
+      ScoutSemaphore.create_semaphore(@write_sem,1)
+      ScoutSemaphore.create_semaphore(@read_sem,1)
+    end
+
+    def clean
+      @cleaned = true
+      @sread.close unless @sread.closed?
+      @swrite.close unless @swrite.closed?
+      Log.low "Destroying socket semaphores: #{[@key] * ", "}"
+      ScoutSemaphore.delete_semaphore(@write_sem)
+      ScoutSemaphore.delete_semaphore(@read_sem)
+    end
+
+
+    def dump(obj)
+      stream = @swrite
+      obj.concurrent_stream = nil if obj.respond_to?(:concurrent_stream)
+      case obj
+      when Integer
+        size_head = [obj,"I"].pack 'La'
+        str = size_head
+      when nil
+        size_head = [0,"N"].pack 'La'
+        str = size_head
+      when String
+        payload = obj
+        size_head = [payload.bytesize,"C"].pack 'La'
+        str = size_head << payload
+      else
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
+      end
+
+      write_length = str.length
+      wrote = stream.write(str) 
+      while wrote < write_length
+        wrote += stream.write(str[wrote..-1]) 
+      end
+    end
+
+    def load
+      stream = @sread
+      size_head = Open.read_stream stream, 5
+
+      size, type = size_head.unpack('La')
+
+      return nil if type == "N"
+      return size.to_i if type == "I"
+      begin
+        payload = Open.read_stream stream, size
+        case type
+        when "S"
+          begin
+            @serializer.load(payload)
+          rescue Exception
+            Log.exception $!
+            raise $!
+          end
+        when "C"
+          payload
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+
+    def closed_read?
+      @sread.closed?
+    end
+
+    def closed_write?
+      @swrite.closed?
+    end
+
+    def close_write
+      self.dump ClosedStream.new
+      @swrite.close unless closed_write?
+    end
+
+    def close_read
+      @sread.close unless closed_read?
+    end
+
+    #{{{ ACCESSOR
+    def push(obj)
+      ScoutSemaphore.synchronize(@write_sem) do
+        self.dump(obj)
+      end
+    end
+
+    def pop
+      ScoutSemaphore.synchronize(@read_sem) do
+        res = self.load
+        raise res if ClosedStream === res
+        res
+      end
+    end
+
+    alias write push
+
+    alias read pop
+  end
+end

data/lib/scout/work_queue/worker.rb

@@ -0,0 +1,59 @@
+class WorkQueue
+  class Worker
+    attr_accessor :pid, :ignore_ouput
+    def initialize(ignore_ouput = false)
+      @ignore_output = ignore_ouput
+    end
+
+    def run
+      @pid = Process.fork do
+        Log.debug "Worker start with #{Process.pid}"
+        yield
+      end
+    end
+
+    def process(input, output = nil, &block)
+      run do
+        begin
+          while obj = input.read
+            if DoneProcessing === obj
+              output.write DoneProcessing.new
+              raise obj 
+            end
+            res = block.call obj
+            output.write res unless output.nil? || ignore_ouput || res == :ignore 
+          end
+        rescue DoneProcessing
+        rescue Interrupt
+        rescue Exception
+          output.write WorkerException.new($!, Process.pid)
+          exit -1
+        end
+      end
+    end
+
+    def abort
+      begin
+        Log.log "Aborting worker #{@pid}"
+        Process.kill "INT", @pid 
+      rescue Errno::ECHILD
+      end
+    end
+
+    def join
+      Log.log "Joining worker #{@pid}"
+      Process.waitpid @pid
+    end
+
+    def self.join(workers)
+      workers = [workers] unless Array === workers
+      begin
+        while pid = Process.wait 
+          status = $?
+            worker = workers.select{|w| w.pid == pid }.first
+        end
+      rescue Errno::ECHILD
+      end
+    end
+  end
+end

data/lib/scout/work_queue.rb

@@ -0,0 +1,113 @@
+require_relative 'work_queue/socket'
+require_relative 'work_queue/worker'
+
+class WorkQueue
+  attr_accessor :workers, :worker_proc, :callback
+
+  def initialize(workers = 0, &block)
+    @input = WorkQueue::Socket.new
+    @output = WorkQueue::Socket.new
+    @workers = workers.times.collect{ Worker.new }
+    @worker_proc = block
+    @worker_mutex = Mutex.new
+    @removed_workers = []
+  end
+
+  def add_worker(&block)
+    worker = Worker.new
+    @worker_mutex.synchronize do
+      @workers.push(worker)
+      if block_given?
+        worker.process @input, @output, &block
+      else
+        worker.process @input, @output, &@worker_proc
+      end
+    end
+    worker
+  end
+
+  def ignore_ouput
+    @workers.each{|w| w.ignore_ouput = true }
+  end
+
+  def remove_one_worker
+    @input.write DoneProcessing.new
+  end
+
+  def remove_worker(pid)
+    @worker_mutex.synchronize do
+      @workers.delete_if{|w| w.pid == pid }
+      @removed_workers << pid
+    end
+  end
+
+  def process(&callback)
+    @reader = Thread.new do |parent|
+      begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Output reader #{Process.pid}"
+        @done_workers ||= []
+        while true
+          obj = @output.read
+          if DoneProcessing === obj
+            done = @worker_mutex.synchronize do
+              Log.low "Worker #{obj.pid} done"
+              @done_workers << obj.pid
+              @done_workers.length == @removed_workers.length + @workers.length
+            end
+            break if done
+          elsif Exception === obj
+            raise obj
+          else
+            callback.call obj if callback
+          end
+        end
+      rescue DoneProcessing
+      rescue Aborted
+      rescue WorkerException
+        Log.error "Exception in worker #{obj.pid} #{Log.fingerprint obj.exception}"
+        self.abort
+        raise obj.exception
+      end
+    end
+
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    Thread.pass until @reader["name"]
+
+    @waiter = Thread.new do
+      begin
+        Thread.current.report_on_exception = false
+        Thread.current["name"] = "Worker waiter #{Process.pid}"
+        while true
+          pid = Process.wait
+          remove_worker(pid)
+          break if workers.empty?
+        end
+      end
+    end
+
+    Thread.pass until @waiter["name"]
+  end
+
+  def write(obj)
+    @input.write obj
+  end
+
+  def abort
+    workers.each{|w| w.abort }
+  end
+
+  def close
+    @worker_mutex.synchronize{ @workers.length }.times do
+      @input.write DoneProcessing.new()
+    end
+  end
+
+  def join
+    @waiter.join if @waiter
+    @reader.join if @reader
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -55,9 +55,9 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
     else
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
     end
   end
 

data/lib/scout/workflow/step.rb

@@ -109,7 +109,8 @@ class Step
   end
 
   def clean
-    FileUtils.rm path.find if path.exist?
+    Open.rm path if Open.exist?(path)
+    Open.rm info_file if Open.exist?(info_file)
   end
 
   def recursive_clean

data/lib/scout/workflow/task.rb

@@ -143,8 +143,8 @@ module Task
     non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
 
     if non_default_inputs.any?
-      hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
-      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
+      hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
+      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
       id = [id, hash] * "_"
     end
 

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 5.2.0 ruby lib
+# stub: scout-gear 7.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "5.2.0"
+  s.version = "7.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-04-28"
+  s.date = "2023-05-01"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -69,6 +69,7 @@ Gem::Specification.new do |s|
     "lib/scout/resource/produce/rake.rb",
     "lib/scout/resource/scout.rb",
     "lib/scout/resource/util.rb",
+    "lib/scout/semaphore.rb",
     "lib/scout/simple_opt.rb",
     "lib/scout/simple_opt/accessor.rb",
     "lib/scout/simple_opt/doc.rb",
@@ -76,6 +77,11 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/parse.rb",
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
+    "lib/scout/tsv.rb",
+    "lib/scout/tsv/parser.rb",
+    "lib/scout/work_queue.rb",
+    "lib/scout/work_queue/socket.rb",
+    "lib/scout/work_queue/worker.rb",
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
     "lib/scout/workflow/documentation.rb",
@@ -96,8 +102,11 @@ Gem::Specification.new do |s|
     "scout_commands/workflow/list",
     "scout_commands/workflow/task",
     "scout_commands/workflow/task_old",
+    "share/color/color_names",
+    "share/color/diverging_colors.hex",
     "test/scout/indiferent_hash/test_case_insensitive.rb",
     "test/scout/indiferent_hash/test_options.rb",
+    "test/scout/log/test_color.rb",
     "test/scout/log/test_progress.rb",
     "test/scout/misc/test_digest.rb",
     "test/scout/misc/test_filesystem.rb",
@@ -128,8 +137,14 @@ Gem::Specification.new do |s|
     "test/scout/test_path.rb",
     "test/scout/test_persist.rb",
     "test/scout/test_resource.rb",
+    "test/scout/test_semaphore.rb",
     "test/scout/test_tmpfile.rb",
+    "test/scout/test_tsv.rb",
+    "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
+    "test/scout/tsv/test_parser.rb",
+    "test/scout/work_queue/test_socket.rb",
+    "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
     "test/scout/workflow/task/test_inputs.rb",