scout-gear 7.3.0 → 8.0.0

data/lib/scout/tsv/util/select.rb

@@ -1,4 +1,70 @@
 module TSV
+  def self.select(key, values, method, fields: nil, field: nil, invert: false, type: nil, sep: nil, &block)
+    return ! select(key, values, method, field: field, invert: false, type: type, sep: sep, &block) if invert
+
+    return yield(key, values) if method.nil? && block_given
+
+    if Hash === method
+      if method.include?(:invert)
+        method = method.dup
+        invert = method.delete(:invert)
+        return select(key, values, method, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+      end
+      field = method.keys.first
+      value = method[field]
+      return select(key, values, value, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+    end
+
+    if field
+      field = fields.index(field) if fields && String === field
+      set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
+    else
+      set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
+    end
+
+    if Array === set
+      set.flatten!
+    else
+      set = [set]
+    end
+
+    case method
+    when Array
+      (method & set).any?
+    when Regexp
+      set.select{|v| v =~ method }.any?
+    when Symbol
+      set.first.send(method)
+    when Numeric
+      set.size > method
+    when String
+      if block_given?
+        field = method
+        field = fields.index?(field) if fields && String === field
+        case 
+        when block.arity == 1
+          if (method == key_field or method == :key)
+            yield(key)
+          else
+            yield(values[method])
+          end
+        when block.arity == 2
+          if (method == key_field or method == :key)
+            yield(key, key)
+          else
+            yield(key, values[method])
+          end
+        end
+      elsif m = method.match(/^([<>]=?)(.*)/)
+        set.select{|v| v.to_f.send($1, $2.to_f) }.any?
+      else
+        set.select{|v| v == method }.any?
+      end
+    when Proc
+      set.select{|v| method.call(v) }.any?
+    end
+  end
+
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
@@ -33,7 +99,7 @@ module TSV
           new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
         end
       end
-    when (String === method || Symbol === method)
+    when ((String === method) || (Symbol === method))
       if block_given?
         case 
         when block.arity == 1
@@ -88,11 +154,11 @@ module TSV
       key  = method.keys.first
       method = method.values.first
       case
-      when (Array === method and (key == :key or key_field == key))
+      when ((Array === method) and (key == :key or key_field == key))
         with_unnamed do
-          Annotated.purge(method).each{|key| 
-            new[key] = self[key] if invert ^ (self.include? key)
-          }
+          keys.each do |key|
+            new[key] = self[key] if invert ^ (method.include? key)
+          end
         end
       when Array === method
         with_unnamed do
@@ -125,7 +191,7 @@ module TSV
           end
         end
 
-      when (String === method and method =~ /name:(.*)/)
+      when ((String === method) and (method =~ /name:(.*)/))
         name = $1
         old_unnamed = self.unnamed
         self.unnamed = false
@@ -187,34 +253,13 @@ module TSV
     new
   end
 
-  def reorder(key_field = nil, fields = nil, merge: true, one2one: :fill) 
-    res = self.annotate({})
-    key_field_name, field_names = through key_field, fields, one2one: one2one do |k,v|
-      if @type == :double && merge && res.include?(k)
-        current = res[k]
-        if merge == :concat
-          v.each_with_index do |new,i|
-            next if new.empty?
-            current[i].concat(new)
-          end
-        else
-          merged = []
-          v.each_with_index do |new,i|
-            next if new.empty?
-            merged[i] = current[i] + new
-          end
-          res[k] = merged
-        end
-      else
-        res[k] = v
+  def subset(keys)
+    new = self.annotate({})
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k] if self.include?(k)
       end
     end
-    res.key_field = key_field_name
-    res.fields = field_names
-    res
-  end
-
-  def slice(fields)
-    reorder :key, fields
+    new
   end
 end

data/lib/scout/tsv/util/unzip.rb

@@ -0,0 +1,86 @@
+module TSV
+
+  def self.unzip(source, field, target: nil, sep: ":", delete: true, type: :list, merge: false, one2one: true, bar: nil)
+    source = TSV::Parser.new source if String === source
+
+    field_pos = source.identify_field(field)
+    new_fields = source.fields.dup
+    field_name = new_fields[field_pos]
+    new_fields.delete_at(field_pos) if delete
+    new_key_field = [source.key_field, field_name] * sep
+    type = :double if merge
+
+    stream = target == :stream
+
+    target = case target
+             when :stream
+               TSV::Dumper.new(source.options.merge(sep: "\t"))
+             when nil
+               TSV.setup({})
+             else
+               target
+             end
+               
+    target.fields = new_fields
+    target.key_field = new_key_field
+    target.type = type
+
+    transformer = TSV::Transformer.new source, target, unnamed: true
+
+    bar = "Unzip #{new_key_field}" if TrueClass === bar
+
+    transformer.traverse unnamed: true, one2one: one2one, bar: bar do |k,v|
+      if source.type == :double
+        if one2one
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            if delete
+              new_values = _v.dup
+              new_values.delete_at field_pos
+            else
+              new_values = _v
+            end
+
+            new_key = [k,field_value] * sep
+            new_values = new_values.collect{|e| [e] } if transformer.type == :double
+            [new_key, new_values]
+          end
+        else
+          all_values = v.collect{|e| e.dup }
+          all_values.delete_at field_pos if delete
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            new_key = [k,field_value] * sep
+            new_values = all_values if transformer.type == :double
+            [new_key, new_values]
+          end
+        end
+        
+        MultipleResult.setup(res)
+      else
+        field_value = v[field_pos]
+
+        if delete
+          new_values = v.dup
+          new_values.delete_at field_pos
+        else
+          new_values = v
+        end
+
+        new_key = [k,field_value] * sep
+
+        new_values = new_values.collect{|e| [e] } if transformer.type == :double
+
+        [new_key, new_values]
+      end
+    end
+
+    stream ? transformer : transformer.tsv(merge: merge)
+  end
+
+  def unzip(*args, **kwargs)
+    TSV.unzip(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/util.rb

@@ -1,21 +1,57 @@
 #require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
 #Log.warn "USING OLD RBBT CODE: #{__FILE__}"
 require_relative 'traverse'
+require_relative 'util/filter'
 require_relative 'util/process'
 require_relative 'util/select'
+require_relative 'util/reorder'
+require_relative 'util/unzip'
 module TSV
-  def [](*args)
-    v = super(*args)
-    NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+  def self.identify_field(key_field, fields, name, strict: nil)
+    return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
+    name.collect!{|n| key_field == n ? :key : n } if Array === name
+    NamedArray.identify_name(fields, name, strict: strict)
+  end
+
+  def identify_field(name, strict: nil)
+    TSV.identify_field(@key_field, @fields, name, strict: strict)
+  end
+
+  def [](key, *rest)
+    v = super(key, *rest)
+    NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
     v
   end
 
+  def options
+    extension_attr_hash
+  end
+
+  def zip_new(key, values, insitu: :lax)
+    values = values.collect{|v| Array === v ? v : [v] } unless Array === values.first
+    if current_values = self[key]
+      if insitu == :lax
+        self[key] = NamedArray.add_zipped(current_values, values)
+      elsif insitu
+        NamedArray.add_zipped(current_values, values)
+      else
+        self[key] = NamedArray.add_zipped(current_values.dup, values)
+      end
+    else
+      if insitu && insitu != :lax
+        self[key] = values.dup
+      else
+        self[key] = values
+      end
+    end
+  end
+
   def each(*args, &block)
     if block_given?
       super(*args) do |k,v|
-      NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
-      block.call(k, v)
-    end
+        NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+        block.call(k, v)
+      end
     else
       super(*args)
     end
@@ -33,13 +69,13 @@ module TSV
     end
   end
 
-  def with_unnamed
+  def with_unnamed(unnamed = true)
     begin
-      old_unnamed = unnamed
-      unnamed = true
+      old_unnamed = @unnamed
+      @unnamed = unnamed
       yield
     ensure
-      unnamed = old_unnamed
+      @unnamed = old_unnamed
     end
   end
 
@@ -53,7 +89,7 @@ module TSV
     end
 
     filename = @filename
-    filename = "No filename" if filename.nil? || filename.empty?
+    filename = "No filename" if filename.nil? || String === filename && filename.empty?
     filename.find if Path === filename 
     filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
 
@@ -73,10 +109,23 @@ Example:
   end
 
   def all_fields
+    return [] if @fields.nil?
     [@key_field] + @fields
   end
 
+  def options
+    self.extension_attr_hash
+  end
+
   def fingerprint
     "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
   end
+
+  def digest_str
+    fingerprint
+  end
+
+  def inspect
+    fingerprint
+  end
 end

data/lib/scout/tsv.rb

@@ -2,23 +2,46 @@ require_relative 'meta_extension'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'
+require_relative 'tsv/transformer'
 require_relative 'tsv/persist'
 require_relative 'tsv/index'
 require_relative 'tsv/path'
 require_relative 'tsv/traverse'
 require_relative 'tsv/open'
+require_relative 'tsv/attach'
+require_relative 'tsv/change_id'
 
 module TSV
   extend MetaExtension
   extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
 
+  def self.str2options(str)
+    field_options,_sep, rest =  str.partition("#")
+    key, fields_str = field_options.split("~")
+
+    fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
+
+    rest = ":type=" << rest if rest =~ /^:?\w+$/
+    rest_options = rest.nil? ? {} : IndiferentHash.string2hash(rest)
+
+    {:key_field => key, :fields => fields}.merge(rest_options)
+  end
+
+  def self.str_setup(option_str, obj)
+    options = TSV.str2options(option_str) 
+    setup(obj, options)
+  end
+
   def self.open(file, options = {})
-    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
-    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+    persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
+    type = type.to_sym if type
+    file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
+    Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
       data = filename ? ScoutCabinet.open(filename, true, type) : nil
       options[:data] = data if data
       options[:filename] = file
-      Open.open(file) do |f|
+      Log.debug "TSV open #{Log.fingerprint file}"
+      Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
         TSV.parse(f, **options)
       end
     end

data/lib/scout/work_queue/socket.rb

@@ -3,7 +3,7 @@ require 'scout/semaphore'
 require 'scout/exceptions'
 class WorkQueue
   class Socket
-    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned, :exception
     def initialize(serializer = nil)
       @sread, @swrite = Open.pipe
 
@@ -112,6 +112,11 @@ class WorkQueue
       end
     end
 
+    def abort(exception)
+      @exception = exception
+      @swrite.close unless closed_write?
+    end
+
     alias write push
 
     alias read pop

data/lib/scout/work_queue/worker.rb

@@ -27,8 +27,10 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          Kernel.exit -1
+          Process.exit! -1
+        ensure
         end
+        Process.exit! 0
       end
     end
 
@@ -36,7 +38,8 @@ class WorkQueue
       begin
         Log.debug "Aborting worker #{@pid}"
         Process.kill "INT", @pid 
-      rescue Errno::ECHILD
+      rescue Errno::ECHILD 
+      rescue Errno::ESRCH
       end
     end
 

data/lib/scout/work_queue.rb

@@ -46,7 +46,11 @@ class WorkQueue
   end
 
   def process(&callback)
-    @reader = Thread.new do |parent|
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+
+    @reader = Thread.new(Thread.current) do |parent|
       begin
         Thread.current.report_on_exception = false
         Thread.current["name"] = "Output reader #{Process.pid}"
@@ -71,8 +75,9 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
+        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
         self.abort
+        @input.abort obj.worker_exception
         raise obj.worker_exception
       rescue
         Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
@@ -81,10 +86,6 @@ class WorkQueue
       end
     end
 
-    @workers.each do |w| 
-      w.process @input, @output, &@worker_proc
-    end
-
     Thread.pass until @reader["name"]
 
     @waiter = Thread.new do
@@ -104,7 +105,13 @@ class WorkQueue
   end
 
   def write(obj)
-    @input.write obj
+    begin
+      @input.write obj
+    rescue Exception
+      raise $! unless @input.exception
+    ensure
+      raise @input.exception if @input.exception
+    end
   end
 
   def abort
@@ -117,7 +124,7 @@ class WorkQueue
   def close
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
-      @input.write DoneProcessing.new()
+      @input.write DoneProcessing.new() unless @input.closed_write?
     end
   end
 

data/lib/scout/workflow/definition.rb

@@ -76,9 +76,11 @@ module Workflow
       end
     when 1
       task = args.first
+      options, task = task, nil if Hash === task
     end
     workflow = self if workflow.nil?
     options = {} if options.nil?
+    task = task.to_sym if task
     annotate_next_task :deps, [workflow, task, options, block, args]
   end
 
@@ -103,6 +105,20 @@ module Workflow
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
+      @annotate_next_task[:extension] ||=  
+        case type
+        when :tsv
+          "tsv"
+        when :yaml
+          "yaml"
+        when :marshal
+          "marshal"
+        when :json
+          "json"
+        else
+          nil
+        end
+
       task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
       @tasks[name] = task
     ensure
@@ -110,11 +126,14 @@ module Workflow
     end
   end
 
+  FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
+  REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns 
-    task name => nil do
+    returns workflow.tasks[oname].returns if @returns.nil?
+    type = workflow.tasks[oname].type 
+    task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last

data/lib/scout/workflow/step/dependencies.rb

@@ -1,8 +1,13 @@
 class Step
+  def rec_dependencies
+    rec_dependencies = dependencies.dup
+    dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
+  end
+
   def recursive_inputs
-    dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
-      acc.concat(dep.inputs) if dep.inputs
-      acc
+    recursive_inputs = @inputs.to_hash
+    dependencies.inject(recursive_inputs) do |acc,dep|
+      acc.merge(dep.recursive_inputs)
     end
   end
 
@@ -16,6 +21,10 @@ class Step
   def prepare_dependencies
     inverse_dep = {}
     dependencies.each{|dep| 
+      if dep.present? && ! dep.updated? 
+        Log.debug "Clean outdated #{dep.path}"
+        dep.clean
+      end
       next if dep.done?
       if dep.dependencies
         dep.dependencies.each do |d|
@@ -34,7 +43,18 @@ class Step
   end
 
   def run_dependencies
-    dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
+    dependencies.each{|dep| dep.run(true) unless dep.running? || dep.done? }
   end
 
+  def abort_dependencies
+    dependencies.each{|dep| dep.abort if dep.running? }
+  end
+
+  def self.wait_for_jobs(jobs)
+    threads = []
+    jobs.each do |job| 
+      threads << job.join
+    end
+    threads.each do |t| t.join end
+  end
 end

data/lib/scout/workflow/step/info.rb

@@ -1,6 +1,7 @@
 class Step
   SERIALIZER = :marshal
   def info_file
+    return nil if @path.nil?
     @info_file ||= begin
                      info_file = @path + ".info"
                      @path.annotate info_file if Path === @path
@@ -19,6 +20,10 @@ class Step
     @info_load_time = Time.now
   end
 
+  def clear_info
+    save_info(@info = {})
+  end
+
   def info
     outdated = begin
                  @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
@@ -36,7 +41,19 @@ class Step
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
-      report_status new_info[:status], new_info[:message] if key == :status
+      if key == :status
+        message = new_info[:messages]
+        if message.nil? && value == :done || value == :error || value == :aborted
+          start = info[:start]
+          eend = new_info[:end]
+          if start && eend
+            time = eend - start
+            time_str = Misc.format_seconds_short(time)
+            message = Log.color(:time, time_str)
+          end
+        end
+        report_status value, message 
+      end
       if Exception === value
         begin
           Marshal.dump(value)
@@ -72,15 +89,21 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
+      Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
-      Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
+      Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end
 
-  def log(status, message = nil)
+  def log(status, message = nil, &block)
+    if block_given?
+      time = Misc.exec_time &block
+      time_str = Misc.format_seconds_short time
+      message = message.nil? ? Log.color(:time, time_str) : "#{Log.color :time, time_str} - #{ message }"
+    end
+
     if message
-      merge_info :status => status, :messages => [message]
+      merge_info :status => status, :messages => message
     else
       merge_info :status => status
     end
@@ -105,4 +128,12 @@ class Step
   def exception
     info[:exception]
   end
+
+  def marshal_dump
+    @path
+  end
+
+  def marshal_load(path)
+    Step.new path
+  end
 end