scout-gear 7.3.0 → 8.1.0

data/lib/scout/tsv/util/select.rb

@@ -1,4 +1,70 @@
 module TSV
+  def self.select(key, values, method, fields: nil, field: nil, invert: false, type: nil, sep: nil, &block)
+    return ! select(key, values, method, field: field, invert: false, type: type, sep: sep, &block) if invert
+
+    return yield(key, values) if method.nil? && block_given
+
+    if Hash === method
+      if method.include?(:invert)
+        method = method.dup
+        invert = method.delete(:invert)
+        return select(key, values, method, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+      end
+      field = method.keys.first
+      value = method[field]
+      return select(key, values, value, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+    end
+
+    if field
+      field = fields.index(field) if fields && String === field
+      set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
+    else
+      set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
+    end
+
+    if Array === set
+      set.flatten!
+    else
+      set = [set]
+    end
+
+    case method
+    when Array
+      (method & set).any?
+    when Regexp
+      set.select{|v| v =~ method }.any?
+    when Symbol
+      set.first.send(method)
+    when Numeric
+      set.size > method
+    when String
+      if block_given?
+        field = method
+        field = fields.index?(field) if fields && String === field
+        case 
+        when block.arity == 1
+          if (method == key_field or method == :key)
+            yield(key)
+          else
+            yield(values[method])
+          end
+        when block.arity == 2
+          if (method == key_field or method == :key)
+            yield(key, key)
+          else
+            yield(key, values[method])
+          end
+        end
+      elsif m = method.match(/^([<>]=?)(.*)/)
+        set.select{|v| v.to_f.send($1, $2.to_f) }.any?
+      else
+        set.select{|v| v == method }.any?
+      end
+    when Proc
+      set.select{|v| method.call(v) }.any?
+    end
+  end
+
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
@@ -33,7 +99,7 @@ module TSV
           new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
         end
       end
-    when (String === method || Symbol === method)
+    when ((String === method) || (Symbol === method))
       if block_given?
         case 
         when block.arity == 1
@@ -88,11 +154,11 @@ module TSV
       key  = method.keys.first
       method = method.values.first
       case
-      when (Array === method and (key == :key or key_field == key))
+      when ((Array === method) and (key == :key or key_field == key))
         with_unnamed do
-          Annotated.purge(method).each{|key| 
-            new[key] = self[key] if invert ^ (self.include? key)
-          }
+          keys.each do |key|
+            new[key] = self[key] if invert ^ (method.include? key)
+          end
         end
       when Array === method
         with_unnamed do
@@ -125,7 +191,7 @@ module TSV
           end
         end
 
-      when (String === method and method =~ /name:(.*)/)
+      when ((String === method) and (method =~ /name:(.*)/))
         name = $1
         old_unnamed = self.unnamed
         self.unnamed = false
@@ -187,34 +253,13 @@ module TSV
     new
   end
 
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: :fill) 
-    res = self.annotate({})
-    key_field_name, field_names = through key_field, fields, one2one: one2one do |k,v|
-      if @type == :double && merge && res.include?(k)
-        current = res[k]
-        if merge == :concat
-          v.each_with_index do |new,i|
-            next if new.empty?
-            current[i].concat(new)
-          end
-        else
-          merged = []
-          v.each_with_index do |new,i|
-            next if new.empty?
-            merged[i] = current[i] + new
-          end
-          res[k] = merged
-        end
-      else
-        res[k] = v
+  def subset(keys)
+    new = self.annotate({})
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k] if self.include?(k)
       end
     end
-    res.key_field = key_field_name
-    res.fields = field_names
-    res
-  end
-
-  def slice(fields)
-    reorder :key, fields
+    new
   end
 end

data/lib/scout/tsv/util/unzip.rb

@@ -0,0 +1,86 @@
+module TSV
+
+  def self.unzip(source, field, target: nil, sep: ":", delete: true, type: :list, merge: false, one2one: true, bar: nil)
+    source = TSV::Parser.new source if String === source
+
+    field_pos = source.identify_field(field)
+    new_fields = source.fields.dup
+    field_name = new_fields[field_pos]
+    new_fields.delete_at(field_pos) if delete
+    new_key_field = [source.key_field, field_name] * sep
+    type = :double if merge
+
+    stream = target == :stream
+
+    target = case target
+             when :stream
+               TSV::Dumper.new(source.options.merge(sep: "\t"))
+             when nil
+               TSV.setup({})
+             else
+               target
+             end
+               
+    target.fields = new_fields
+    target.key_field = new_key_field
+    target.type = type
+
+    transformer = TSV::Transformer.new source, target, unnamed: true
+
+    bar = "Unzip #{new_key_field}" if TrueClass === bar
+
+    transformer.traverse unnamed: true, one2one: one2one, bar: bar do |k,v|
+      if source.type == :double
+        if one2one
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            if delete
+              new_values = _v.dup
+              new_values.delete_at field_pos
+            else
+              new_values = _v
+            end
+
+            new_key = [k,field_value] * sep
+            new_values = new_values.collect{|e| [e] } if transformer.type == :double
+            [new_key, new_values]
+          end
+        else
+          all_values = v.collect{|e| e.dup }
+          all_values.delete_at field_pos if delete
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            new_key = [k,field_value] * sep
+            new_values = all_values if transformer.type == :double
+            [new_key, new_values]
+          end
+        end
+        
+        MultipleResult.setup(res)
+      else
+        field_value = v[field_pos]
+
+        if delete
+          new_values = v.dup
+          new_values.delete_at field_pos
+        else
+          new_values = v
+        end
+
+        new_key = [k,field_value] * sep
+
+        new_values = new_values.collect{|e| [e] } if transformer.type == :double
+
+        [new_key, new_values]
+      end
+    end
+
+    stream ? transformer : transformer.tsv(merge: merge)
+  end
+
+  def unzip(*args, **kwargs)
+    TSV.unzip(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/util.rb

@@ -1,21 +1,57 @@
 #require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
 #Log.warn "USING OLD RBBT CODE: #{__FILE__}"
 require_relative 'traverse'
+require_relative 'util/filter'
 require_relative 'util/process'
 require_relative 'util/select'
+require_relative 'util/reorder'
+require_relative 'util/unzip'
 module TSV
-  def [](*args)
-    v = super(*args)
-    NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+  def self.identify_field(key_field, fields, name, strict: nil)
+    return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
+    name.collect!{|n| key_field == n ? :key : n } if Array === name
+    NamedArray.identify_name(fields, name, strict: strict)
+  end
+
+  def identify_field(name, strict: nil)
+    TSV.identify_field(@key_field, @fields, name, strict: strict)
+  end
+
+  def [](key, *rest)
+    v = super(key, *rest)
+    NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
     v
   end
 
+  def options
+    extension_attr_hash
+  end
+
+  def zip_new(key, values, insitu: :lax)
+    values = values.collect{|v| Array === v ? v : [v] } unless Array === values.first
+    if current_values = self[key]
+      if insitu == :lax
+        self[key] = NamedArray.add_zipped(current_values, values)
+      elsif insitu
+        NamedArray.add_zipped(current_values, values)
+      else
+        self[key] = NamedArray.add_zipped(current_values.dup, values)
+      end
+    else
+      if insitu && insitu != :lax
+        self[key] = values.dup
+      else
+        self[key] = values
+      end
+    end
+  end
+
   def each(*args, &block)
     if block_given?
       super(*args) do |k,v|
-      NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
-      block.call(k, v)
-    end
+        NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+        block.call(k, v)
+      end
     else
       super(*args)
     end
@@ -33,13 +69,13 @@ module TSV
     end
   end
 
-  def with_unnamed
+  def with_unnamed(unnamed = true)
     begin
-      old_unnamed = unnamed
-      unnamed = true
+      old_unnamed = @unnamed
+      @unnamed = unnamed
       yield
     ensure
-      unnamed = old_unnamed
+      @unnamed = old_unnamed
     end
   end
 
@@ -53,7 +89,7 @@ module TSV
     end
 
     filename = @filename
-    filename = "No filename" if filename.nil? || filename.empty?
+    filename = "No filename" if filename.nil? || String === filename && filename.empty?
     filename.find if Path === filename 
     filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
 
@@ -73,10 +109,23 @@ Example:
   end
 
   def all_fields
+    return [] if @fields.nil?
     [@key_field] + @fields
   end
 
+  def options
+    self.extension_attr_hash
+  end
+
   def fingerprint
     "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
   end
+
+  def digest_str
+    fingerprint
+  end
+
+  def inspect
+    fingerprint
+  end
 end

data/lib/scout/tsv.rb

@@ -2,23 +2,53 @@ require_relative 'meta_extension'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'
+require_relative 'tsv/transformer'
 require_relative 'tsv/persist'
 require_relative 'tsv/index'
 require_relative 'tsv/path'
 require_relative 'tsv/traverse'
 require_relative 'tsv/open'
+require_relative 'tsv/attach'
+require_relative 'tsv/change_id'
+require_relative 'tsv/stream'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
+  extension_attr :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers
+
+  def self.str2options(str)
+    field_options,_sep, rest =  str.partition("#")
+    key, fields_str = field_options.split("~")
+
+    fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
+
+    rest = ":type=" << rest if rest =~ /^:?\w+$/
+    rest_options = rest.nil? ? {} : IndiferentHash.string2hash(rest)
+
+    {:key_field => key, :fields => fields}.merge(rest_options)
+  end
+
+  def self.str_setup(option_str, obj)
+    options = TSV.str2options(option_str) 
+    setup(obj, options)
+  end
 
   def self.open(file, options = {})
-    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
-    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+    persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
+    type = type.to_sym if type
+    file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
+    Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
       data = filename ? ScoutCabinet.open(filename, true, type) : nil
       options[:data] = data if data
       options[:filename] = file
-      Open.open(file) do |f|
+
+      if data
+        Log.debug "TSV open #{Log.fingerprint file} into #{Log.fingerprint data}"
+      else
+        Log.debug "TSV open #{Log.fingerprint file}"
+      end
+
+      Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
         TSV.parse(f, **options)
       end
     end

data/lib/scout/work_queue/socket.rb

@@ -3,7 +3,7 @@ require 'scout/semaphore'
 require 'scout/exceptions'
 class WorkQueue
   class Socket
-    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned, :exception
     def initialize(serializer = nil)
       @sread, @swrite = Open.pipe
 
@@ -112,6 +112,11 @@ class WorkQueue
       end
     end
 
+    def abort(exception)
+      @exception = exception
+      @swrite.close unless closed_write?
+    end
+
     alias write push
 
     alias read pop

data/lib/scout/work_queue/worker.rb

@@ -27,8 +27,10 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          Kernel.exit -1
+          exit -1
+        ensure
         end
+        exit 0
       end
     end
 
@@ -36,7 +38,8 @@ class WorkQueue
       begin
         Log.debug "Aborting worker #{@pid}"
         Process.kill "INT", @pid 
-      rescue Errno::ECHILD
+      rescue Errno::ECHILD 
+      rescue Errno::ESRCH
       end
     end
 

data/lib/scout/work_queue.rb

@@ -1,5 +1,6 @@
 require_relative 'work_queue/socket'
 require_relative 'work_queue/worker'
+require 'timeout'
 
 class WorkQueue
   attr_accessor :workers, :worker_proc, :callback
@@ -38,7 +39,7 @@ class WorkQueue
     @worker_mutex.synchronize do
       worker = @workers.index{|w| w.pid == pid}
       if worker
-        Log.debug "Removed worker #{pid}"
+        Log.low "Removed worker #{pid}"
         @workers.delete_at(worker)
         @removed_workers << pid
       end
@@ -46,7 +47,11 @@ class WorkQueue
   end
 
   def process(&callback)
-    @reader = Thread.new do |parent|
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    @reader = Thread.new(Thread.current) do |parent|
       begin
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{Process.pid}"
@@ -71,8 +76,9 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
+        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
         self.abort
+        @input.abort obj.worker_exception
         raise obj.worker_exception
       rescue
         Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
@@ -81,30 +87,45 @@ class WorkQueue
       end
     end
 
-    @workers.each do |w| 
-      w.process @input, @output, &@worker_proc
-    end
-
     Thread.pass until @reader["name"]
 
+    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
+
     @waiter = Thread.new do
-      begin
-        Thread.current.report_on_exception = false
-        Thread.current["name"] = "Worker waiter #{Process.pid}"
-        while true
-          pid = Process.wait
-          remove_worker(pid)
-          break if @worker_mutex.synchronize{ @workers.empty? }
+      Thread.current.report_on_exception = false
+      Thread.current["name"] = "Worker waiter #{Process.pid}"
+      while true
+        break if @worker_mutex.synchronize{ @workers.empty? }
+        begin
+          Timeout.timeout(1) do
+            begin
+              pid, status = Process.wait2
+              remove_worker(pid) if pid
+            rescue Exception
+              Log.exception $!
+            end
+          end
+        rescue Timeout::Error
+          pids = @worker_mutex.synchronize{ @workers.collect{|w| w.pid } }
+          pids.each do |p|
+            pid, status = Process.wait2 p, Process::WNOHANG
+            remove_worker(pid) if pid
+          end
         end
       end
     end
 
-    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
     Thread.pass until @waiter["name"]
   end
 
   def write(obj)
-    @input.write obj
+    begin
+      @input.write obj
+    rescue Exception
+      raise $! unless @input.exception
+    ensure
+      raise @input.exception if @input.exception
+    end
   end
 
   def abort
@@ -117,12 +138,22 @@ class WorkQueue
   def close
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
-      @input.write DoneProcessing.new()
+      @input.write DoneProcessing.new() unless @input.closed_write?
     end
   end
 
-  def join
-    @waiter.join if @waiter
-    @reader.join if @reader
+  def clean
+    @waiter.join if @waiter 
+    @input.clean
+    @output.clean
+  end
+
+  def join(clean = true)
+    begin
+      @waiter.join if @waiter
+      @reader.join if @reader
+    ensure
+      self.clean if clean
+    end
   end
 end

data/lib/scout/workflow/definition.rb

@@ -50,7 +50,7 @@ module Workflow
 
   def directory=(directory)
     @directory = directory
-    @tasks.each{|name,d| d.directory = directory[name] } if @tasks
+    @tasks.each{|name,d| d.directory = Path === directory ? directory[name] : File.join(directory, name.to_s) } if @tasks
   end
 
   def annotate_next_task(type, obj)
@@ -76,9 +76,11 @@ module Workflow
       end
     when 1
       task = args.first
+      options, task = task, nil if Hash === task
     end
     workflow = self if workflow.nil?
     options = {} if options.nil?
+    task = task.to_sym if task
     annotate_next_task :deps, [workflow, task, options, block, args]
   end
 
@@ -101,8 +103,23 @@ module Workflow
   def task(name_and_type, &block)
     name, type = name_and_type.collect.first
     @tasks ||= IndiferentHash.setup({})
+    block = self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
+      @annotate_next_task[:extension] ||=  
+        case type
+        when :tsv
+          "tsv"
+        when :yaml
+          "yaml"
+        when :marshal
+          "marshal"
+        when :json
+          "json"
+        else
+          nil
+        end
+
       task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
       @tasks[name] = task
     ensure
@@ -110,11 +127,14 @@ module Workflow
     end
   end
 
+  FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
+  REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns 
-    task name => nil do
+    returns workflow.tasks[oname].returns if @returns.nil?
+    type = workflow.tasks[oname].type 
+    task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last