scout-gear 7.2.0 → 8.0.0

data/lib/scout/tsv/util/reorder.rb

@@ -0,0 +1,81 @@
+require 'matrix'
+
+module TSV
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs) 
+    res = self.annotate({})
+    res.type = kwargs[:type] if kwargs.include?(:type)
+    kwargs[:one2one] = one2one
+    key_field_name, field_names = traverse key_field, fields, **kwargs do |k,v|
+      if @type == :double && merge && res.include?(k)
+        current = res[k]
+        if merge == :concat
+          v.each_with_index do |new,i|
+            next if new.empty?
+            current[i].concat(new)
+          end
+        else
+          merged = []
+          v.each_with_index do |new,i|
+            next if new.empty?
+            merged[i] = current[i] + new
+          end
+          res[k] = merged
+        end
+      else
+        res[k] = v
+      end
+    end
+    res.key_field = key_field_name
+    res.fields = field_names
+    res
+  end
+
+  def slice(fields, **kwargs)
+    reorder :key, fields, **kwargs
+  end
+
+  def column(field, **kwargs)
+    new_type = case type
+               when :double, :flat
+                 :flat
+               else
+                 :single
+               end
+
+    kwargs[:type] = new_type
+    slice(field, **kwargs)
+  end
+
+  def transpose_list(key_field="Unkown ID")
+    new_fields = keys.dup
+    new = self.annotate({})
+    TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    m = Matrix.rows values 
+    new_rows = m.transpose.to_a
+
+    fields.zip(new_rows) do |key,row|
+      new[key] = row
+    end
+
+    new
+  end
+
+  def transpose_double(key_field = "Unkown ID")
+    sep = "-!SEP--#{rand 10000}!-"
+    tmp = self.to_list{|v| v * sep}
+    new = tmp.transpose_list(key_field)
+    new.to_double{|v| v.split(sep)}
+  end
+
+  def transpose(key_field = "Unkown ID")
+    case type
+    when :single, :flat
+      self.to_list.transpose_list key_field
+    when :list
+      transpose_list key_field
+    when :double
+      transpose_double key_field
+    end
+  end
+end

data/lib/scout/tsv/util/select.rb

@@ -0,0 +1,265 @@
+module TSV
+  def self.select(key, values, method, fields: nil, field: nil, invert: false, type: nil, sep: nil, &block)
+    return ! select(key, values, method, field: field, invert: false, type: type, sep: sep, &block) if invert
+
+    return yield(key, values) if method.nil? && block_given
+
+    if Hash === method
+      if method.include?(:invert)
+        method = method.dup
+        invert = method.delete(:invert)
+        return select(key, values, method, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+      end
+      field = method.keys.first
+      value = method[field]
+      return select(key, values, value, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
+    end
+
+    if field
+      field = fields.index(field) if fields && String === field
+      set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
+    else
+      set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
+    end
+
+    if Array === set
+      set.flatten!
+    else
+      set = [set]
+    end
+
+    case method
+    when Array
+      (method & set).any?
+    when Regexp
+      set.select{|v| v =~ method }.any?
+    when Symbol
+      set.first.send(method)
+    when Numeric
+      set.size > method
+    when String
+      if block_given?
+        field = method
+        field = fields.index?(field) if fields && String === field
+        case 
+        when block.arity == 1
+          if (method == key_field or method == :key)
+            yield(key)
+          else
+            yield(values[method])
+          end
+        when block.arity == 2
+          if (method == key_field or method == :key)
+            yield(key, key)
+          else
+            yield(key, values[method])
+          end
+        end
+      elsif m = method.match(/^([<>]=?)(.*)/)
+        set.select{|v| v.to_f.send($1, $2.to_f) }.any?
+      else
+        set.select{|v| v == method }.any?
+      end
+    when Proc
+      set.select{|v| method.call(v) }.any?
+    end
+  end
+
+  def select(method = nil, invert = false, &block)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    self.annotate(new)
+    
+    case
+    when (method.nil? and block_given?)
+      through do |key, values|
+        new[key] = values if invert ^ (yield key, values)
+      end
+    when Array === method
+      method = Set.new method
+      with_unnamed do
+        case type
+        when :single
+          through do |key, value|
+            new[key] = value if invert ^ (method.include? key or method.include? value)
+          end
+        when :list, :flat
+          through do |key, values|
+            new[key] = values if invert ^ (method.include? key or (method & values).length > 0)
+          end
+        else
+          through do |key, values|
+            new[key] = values if invert ^ (method.include? key or (method & values.flatten).length > 0)
+          end
+        end
+      end
+    when Regexp === method
+      with_unnamed do
+        through do |key, values|
+          new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
+        end
+      end
+    when ((String === method) || (Symbol === method))
+      if block_given?
+        case 
+        when block.arity == 1
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key))
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if invert ^ (yield(value))
+              end
+            else
+              pos = identify_field method
+              raise "Field #{ method } not identified. Available: #{ fields * ", " }" if pos.nil?
+
+              through do |key, values|
+                new[key] = values if invert ^ (yield(values[pos]))
+              end
+            end
+          end
+        when block.arity == 2
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key, key))
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if invert ^ (yield(key, value))
+              end
+            else
+              pos = identify_field method
+              through do |key, values|
+                new[key] = values if invert ^ (yield(key, values[pos]))
+              end
+            end
+
+          end
+        end
+
+      else
+        with_unnamed do
+          through do |key, values|
+            new[key] = values if invert ^ ([key,values].flatten.select{|v| v == method}.any?)
+          end
+        end
+      end
+    when Hash === method
+      key  = method.keys.first
+      method = method.values.first
+      case
+      when ((Array === method) and (key == :key or key_field == key))
+        with_unnamed do
+          keys.each do |key|
+            new[key] = self[key] if invert ^ (method.include? key)
+          end
+        end
+      when Array === method
+        with_unnamed do
+          method = Set.new method unless Set === method
+          case type
+          when :single
+            through :key, key do |key, value|
+              new[key] = self[key] if invert ^ (method.include? value)
+            end
+          when :list
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ (method.include? values.first)
+            end
+          when :flat #untested
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
+            end
+          else
+            through :key, key do |key, values|
+              new[key] = self[key] if invert ^ ((method & values.flatten).any?)
+            end
+          end
+        end
+
+      when Regexp === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| v =~ method}.any?)
+          end
+        end
+
+      when ((String === method) and (method =~ /name:(.*)/))
+        name = $1
+        old_unnamed = self.unnamed
+        self.unnamed = false
+        if name.strip =~ /^\/(.*)\/$/
+          regexp = Regexp.new $1
+          through :key, key do |key, values|
+            case type
+            when :single
+              values = values.annotate([values])
+            when :double
+              values = values[0]
+            end
+            new[key] = self[key] if invert ^ (values.select{|v| v.name =~ regexp}.any?)
+          end
+        else
+          through :key, key do |key, values|
+            case type
+            when :single
+              values = values.annotate([values])
+            when :double
+              values = values[0]
+            end
+            new[key] = self[key] if invert ^ (values.select{|v| v.name == name}.any?)
+          end
+        end
+        self.unnamed = old_unnamed
+
+      when String === method
+        if method =~ /^([<>]=?)(.*)/
+          with_unnamed do
+            through :key, key do |key, values|
+              value = Array === values ? values.flatten.first : values
+              new[key] = self[key] if value.to_f.send($1, $2.to_f)
+            end
+          end
+        else
+          with_unnamed do
+            through :key, key do |key, values|
+              values = [values] if type == :single
+              new[key] = self[key] if invert ^ (values.flatten.select{|v| v == method}.length > 0)
+            end
+          end
+        end
+      when Numeric === method
+        with_unnamed do
+          through :key, key do |key, values|
+            new[key] = self[key] if invert ^ (values.flatten.length >= method)
+          end
+        end
+      when Proc === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if invert ^ (values.flatten.select{|v| method.call(v)}.length > 0)
+          end
+        end
+      end
+    end
+    new
+  end
+
+  def subset(keys)
+    new = self.annotate({})
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k] if self.include?(k)
+      end
+    end
+    new
+  end
+end

data/lib/scout/tsv/util/unzip.rb

@@ -0,0 +1,86 @@
+module TSV
+
+  def self.unzip(source, field, target: nil, sep: ":", delete: true, type: :list, merge: false, one2one: true, bar: nil)
+    source = TSV::Parser.new source if String === source
+
+    field_pos = source.identify_field(field)
+    new_fields = source.fields.dup
+    field_name = new_fields[field_pos]
+    new_fields.delete_at(field_pos) if delete
+    new_key_field = [source.key_field, field_name] * sep
+    type = :double if merge
+
+    stream = target == :stream
+
+    target = case target
+             when :stream
+               TSV::Dumper.new(source.options.merge(sep: "\t"))
+             when nil
+               TSV.setup({})
+             else
+               target
+             end
+               
+    target.fields = new_fields
+    target.key_field = new_key_field
+    target.type = type
+
+    transformer = TSV::Transformer.new source, target, unnamed: true
+
+    bar = "Unzip #{new_key_field}" if TrueClass === bar
+
+    transformer.traverse unnamed: true, one2one: one2one, bar: bar do |k,v|
+      if source.type == :double
+        if one2one
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            if delete
+              new_values = _v.dup
+              new_values.delete_at field_pos
+            else
+              new_values = _v
+            end
+
+            new_key = [k,field_value] * sep
+            new_values = new_values.collect{|e| [e] } if transformer.type == :double
+            [new_key, new_values]
+          end
+        else
+          all_values = v.collect{|e| e.dup }
+          all_values.delete_at field_pos if delete
+          res = NamedArray.zip_fields(v).collect do |_v|
+            field_value = _v[field_pos]
+
+            new_key = [k,field_value] * sep
+            new_values = all_values if transformer.type == :double
+            [new_key, new_values]
+          end
+        end
+        
+        MultipleResult.setup(res)
+      else
+        field_value = v[field_pos]
+
+        if delete
+          new_values = v.dup
+          new_values.delete_at field_pos
+        else
+          new_values = v
+        end
+
+        new_key = [k,field_value] * sep
+
+        new_values = new_values.collect{|e| [e] } if transformer.type == :double
+
+        [new_key, new_values]
+      end
+    end
+
+    stream ? transformer : transformer.tsv(merge: merge)
+  end
+
+  def unzip(*args, **kwargs)
+    TSV.unzip(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/util.rb

@@ -1,24 +1,131 @@
 #require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
 #Log.warn "USING OLD RBBT CODE: #{__FILE__}"
+require_relative 'traverse'
+require_relative 'util/filter'
+require_relative 'util/process'
+require_relative 'util/select'
+require_relative 'util/reorder'
+require_relative 'util/unzip'
 module TSV
-  #[:each, :collect, :map].each do |method|
-  #  define_method(method) do |*args,&block|
-  #    super(*args) do |k,v|
-  #      NamedArray.setup(v, @fields) unless @unnamed
-  #      block.call k, v
-  #    end
-  #  end
-  #end
-
-  #[:select, :reject].each do |method|
-  #  define_method(method) do |*args,&block|
-  #    res = super(*args) do |k,v|
-  #      NamedArray.setup(v, @fields) unless @unnamed
-  #      block.call k, v
-  #    end
-  #    self.annotate(res)
-  #    res
-  #  end
-  #end
+  def self.identify_field(key_field, fields, name, strict: nil)
+    return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
+    name.collect!{|n| key_field == n ? :key : n } if Array === name
+    NamedArray.identify_name(fields, name, strict: strict)
+  end
 
+  def identify_field(name, strict: nil)
+    TSV.identify_field(@key_field, @fields, name, strict: strict)
+  end
+
+  def [](key, *rest)
+    v = super(key, *rest)
+    NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
+    v
+  end
+
+  def options
+    extension_attr_hash
+  end
+
+  def zip_new(key, values, insitu: :lax)
+    values = values.collect{|v| Array === v ? v : [v] } unless Array === values.first
+    if current_values = self[key]
+      if insitu == :lax
+        self[key] = NamedArray.add_zipped(current_values, values)
+      elsif insitu
+        NamedArray.add_zipped(current_values, values)
+      else
+        self[key] = NamedArray.add_zipped(current_values.dup, values)
+      end
+    else
+      if insitu && insitu != :lax
+        self[key] = values.dup
+      else
+        self[key] = values
+      end
+    end
+  end
+
+  def each(*args, &block)
+    if block_given?
+      super(*args) do |k,v|
+        NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+        block.call(k, v)
+      end
+    else
+      super(*args)
+    end
+  end
+
+  def collect(*args, &block)
+    if block_given?
+      res = []
+      each do |k,v|
+        res << yield(k, v)
+      end
+      res
+    else
+      super(*args)
+    end
+  end
+
+  def with_unnamed(unnamed = true)
+    begin
+      old_unnamed = @unnamed
+      @unnamed = unnamed
+      yield
+    ensure
+      @unnamed = old_unnamed
+    end
+  end
+
+  def summary
+    key = nil
+    values = nil
+    self.each do |k, v|
+      key = k
+      values = v
+      break
+    end
+
+    filename = @filename
+    filename = "No filename" if filename.nil? || String === filename && filename.empty?
+    filename.find if Path === filename 
+    filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
+
+    with_unnamed do
+      <<-EOF
+Filename = #{filename}
+Key field = #{key_field || "*No key field*"}
+Fields = #{fields ? Log.fingerprint(fields) : "*No field info*"}
+Type = #{type}
+Size = #{size}
+namespace = #{Log.fingerprint namespace}
+identifiers = #{Log.fingerprint identifiers}
+Example:
+  - #{key} -- #{Log.fingerprint values }
+      EOF
+    end
+  end
+
+  def all_fields
+    return [] if @fields.nil?
+    [@key_field] + @fields
+  end
+
+  def options
+    self.extension_attr_hash
+  end
+
+  def fingerprint
+    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
+  end
+
+  def digest_str
+    fingerprint
+  end
+
+  def inspect
+    fingerprint
+  end
 end

data/lib/scout/tsv.rb

@@ -2,26 +2,49 @@ require_relative 'meta_extension'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'
+require_relative 'tsv/transformer'
 require_relative 'tsv/persist'
 require_relative 'tsv/index'
 require_relative 'tsv/path'
 require_relative 'tsv/traverse'
+require_relative 'tsv/open'
+require_relative 'tsv/attach'
+require_relative 'tsv/change_id'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed
+  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
+
+  def self.str2options(str)
+    field_options,_sep, rest =  str.partition("#")
+    key, fields_str = field_options.split("~")
+
+    fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
+
+    rest = ":type=" << rest if rest =~ /^:?\w+$/
+    rest_options = rest.nil? ? {} : IndiferentHash.string2hash(rest)
+
+    {:key_field => key, :fields => fields}.merge(rest_options)
+  end
+
+  def self.str_setup(option_str, obj)
+    options = TSV.str2options(option_str) 
+    setup(obj, options)
+  end
 
   def self.open(file, options = {})
-    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
-    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+    persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
+    type = type.to_sym if type
+    file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
+    Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
       data = filename ? ScoutCabinet.open(filename, true, type) : nil
       options[:data] = data if data
       options[:filename] = file
-      Open.open(file) do |f|
+      Log.debug "TSV open #{Log.fingerprint file}"
+      Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
         TSV.parse(f, **options)
       end
     end
   end
-
 end
 

data/lib/scout/work_queue/socket.rb

@@ -3,7 +3,7 @@ require 'scout/semaphore'
 require 'scout/exceptions'
 class WorkQueue
   class Socket
-    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned, :exception
     def initialize(serializer = nil)
       @sread, @swrite = Open.pipe
 
@@ -112,6 +112,11 @@ class WorkQueue
       end
     end
 
+    def abort(exception)
+      @exception = exception
+      @swrite.close unless closed_write?
+    end
+
     alias write push
 
     alias read pop

data/lib/scout/work_queue/worker.rb

@@ -27,8 +27,10 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          exit -1
+          Process.exit! -1
+        ensure
         end
+        Process.exit! 0
       end
     end
 
@@ -36,7 +38,8 @@ class WorkQueue
       begin
         Log.debug "Aborting worker #{@pid}"
         Process.kill "INT", @pid 
-      rescue Errno::ECHILD
+      rescue Errno::ECHILD 
+      rescue Errno::ESRCH
       end
     end