scout-gear 10.4.0 → 10.6.1

data/lib/scout/tsv/util/reorder.rb

@@ -5,26 +5,36 @@ module TSV
     res = self.annotate({})
     res.type = kwargs[:type] if kwargs.include?(:type)
     kwargs[:one2one] = one2one
-    key_field_name, field_names = traverse key_field, fields, **kwargs do |k,v|
-      if @type == :double && merge && res.include?(k)
-        current = res[k]
-        if merge == :concat
-          v.each_with_index do |new,i|
-            next if new.empty?
-            current[i].concat(new)
+    key_field_name, field_names = with_unnamed do
+      traverse key_field, fields, **kwargs do |k,v|
+        if res.type == :double && merge && res.include?(k)
+          current = res[k]
+          if merge == :concat
+            v.each_with_index do |new,i|
+              next if new.empty?
+              current[i].concat(new)
+            end
+          else
+            merged = []
+            v.each_with_index do |new,i|
+              next if new.empty?
+              merged[i] = current[i] + new
+            end
+            res[k] = merged
           end
-        else
-          merged = []
-          v.each_with_index do |new,i|
-            next if new.empty?
-            merged[i] = current[i] + new
+        elsif res.type == :flat
+          res[k] ||= []
+          if merge == :concat
+            res[k].concat v
+          else
+            res[k] += v
           end
-          res[k] = merged
+        else
+          res[k] = v
         end
-      else
-        res[k] = v
       end
     end
+
     res.key_field = key_field_name
     res.fields = field_names
     res

data/lib/scout/tsv/util/select.rb

@@ -16,7 +16,7 @@ module TSV
     end
 
     if field
-      field = fields.index(field) if fields && String === field
+      field = NamedArray.identify_name(fields, field) if fields && String === field
       set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
     else
       set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
@@ -262,4 +262,12 @@ module TSV
     end
     new
   end
+
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
 end

data/lib/scout/tsv/util/sort.rb

@@ -22,6 +22,79 @@ module TSV
       end
     end
 
+    if not block_given?
+      if fields == :all
+        if just_keys
+          keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
+        else
+          elems.sort_by{|key, value| key }
+        end
+      else
+        sorted = elems.sort do |a, b| 
+          a_value = a.last
+          b_value = b.last
+          a_empty = a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?)
+          b_empty = b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?)
+          case
+          when (a_empty and b_empty)
+            0
+          when a_empty
+            -1
+          when b_empty
+            1
+          when Array === a_value
+            if a_value.length == 1 and b_value.length == 1
+              a_value.first <=> b_value.first
+            else
+              a_value.length <=> b_value.length
+            end
+          else
+            a_value <=> b_value
+          end
+        end
+        if just_keys
+          keys = sorted.collect{|key, value| key}
+          keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
+          keys
+        else
+          sorted.collect{|key, value| [key, self[key]]}
+        end
+      end
+    else
+      if just_keys
+        keys = elems.sort_by(&block).collect{|key, value| key}
+        keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
+        keys
+      else
+        elems.sort_by(&block).collect{|key, value| [key, self[key]]}
+      end
+    end
+  end
+
+  def sort(field = nil, just_keys = false, &block)
+    field = :all if field.nil?
+
+    if field == :all
+      elems = collect
+    else
+      elems = []
+      case type
+      when :single
+        through :key, field do |key, field|
+          elems << [key, field]
+        end
+      when :list, :flat
+        through :key, field do |key, fields|
+          elems << [key, fields.first]
+        end
+      when :double
+        through :key, field do |key, fields|
+          elems << [key, fields.first]
+        end
+      end
+    end
+
     if not block_given?
       if fields == :all
         if just_keys
@@ -63,12 +136,27 @@ module TSV
       end
     else
       if just_keys
-        keys = elems.sort_by(&block).collect{|key, value| key}
+        keys = elems.sort(&block).collect{|key, value| key}
         keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
         keys
       else
-        elems.sort_by(&block).collect{|key, value| [key, self[key]]}
+        elems.sort(&block).collect{|key, value| [key, self[key]]}
       end
     end
   end
+
+  def page(pnum, psize, field = nil, just_keys = false, reverse = false, &block)
+    pstart = psize * (pnum - 1)
+    pend = psize * pnum - 1
+    field = :key if field == "key"
+    keys = sort_by(field || :key, true, &block)
+    keys.reverse! if reverse
+
+    if just_keys
+      keys[pstart..pend]
+    else
+      select :key => keys[pstart..pend]
+    end
+  end
+
 end

data/lib/scout/tsv/util/unzip.rb

@@ -83,4 +83,60 @@ module TSV
   def unzip(*args, **kwargs)
     TSV.unzip(self, *args, **kwargs)
   end
+
+  def unzip_replicates
+    raise "Can only unzip replicates in :double TSVs" unless type == :double
+
+    new = {}
+    self.with_unnamed do
+      through do |k,vs|
+        NamedArray.zip_fields(vs).each_with_index do |v,i|
+          new[k + "(#{i})"] = v
+        end
+      end
+    end
+
+    self.annotate(new)
+    new.type = :list
+
+    new
+  end
+
+  def zip(merge = false, field = "New Field", sep = ":")
+    new = {}
+    self.annotate new
+
+    new.type = :double if merge
+
+    new.with_unnamed do
+      if merge
+        self.through do |key,values|
+          new_key, new_value = key.split(sep)
+          new_values = values + [[new_value] * values.first.length]
+          if new.include? new_key
+            current = new[new_key]
+            current.each_with_index do |v,i|
+              v.concat(new_values[i])
+            end
+          else
+            new[new_key] = new_values
+          end
+        end
+      else
+        self.through do |key,values|
+          new_key, new_value = key.split(sep)
+          new_values = values + [new_value]
+          new[new_key] = new_values
+        end
+      end
+    end
+
+    if self.key_field and self.fields
+      new.key_field = self.key_field.partition(sep).first
+      new.fields = new.fields + [field]
+    end
+
+    new
+  end
+
 end

data/lib/scout/tsv/util.rb

@@ -9,9 +9,43 @@ require_relative 'util/unzip'
 require_relative 'util/sort'
 require_relative 'util/melt'
 module TSV
+
+  def self.field_match_counts(file, values, options = {})
+    options = IndiferentHash.add_defaults options, :persist_prefix => "Field_Matches"
+    persist_options = IndiferentHash.pull_keys options, :persist
+
+    filename = TSV === file ? file.filename : file
+    path = Persist.persist filename, :string, persist_options.merge(:no_load => true) do
+      tsv = TSV === file ? file : TSV.open(file, options)
+
+      text = ""
+      fields = nil
+      tsv.tap{|e| e.unnamed =  true; fields = e.fields}.through do |gene, names|
+        names.zip(fields).each do |list, format|
+          list = [list] unless Array === list
+          list.delete_if do |name| name.empty? end
+          next if list.empty?
+          text << list.collect{|name| [name, format] * "\t"} * "\n" << "\n"
+        end
+        text << [gene, tsv.key_field] * "\t" << "\n"
+      end
+      text
+    end
+
+    TmpFile.with_file(values.uniq * "\n", false) do |value_file|
+      cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2  |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
+      begin
+        TSV.open(CMD.cmd(cmd), :key_field => 1, :fields => [0], :type => :single, :cast => :to_i)
+      rescue
+        Log.exception $!
+        TSV.setup({}, :type => :single, :cast => :to_i)
+      end
+    end
+  end
+
   def self.identify_field(key_field, fields, name, strict: nil)
     return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
-    name.collect!{|n| key_field == n ? :key : n } if Array === name
+    name.collect!{|n| NamedArray.field_match(key_field, n) ? :key : n } if Array === name
     NamedArray.identify_name(fields, name, strict: strict)
   end
 
@@ -21,12 +55,12 @@ module TSV
 
   def [](key, *rest)
     v = super(key, *rest)
-    NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
+    NamedArray.setup(v, @fields, key) unless @unnamed || @type == :flat || ! (Array === v)
     v
   end
 
   def options
-    extension_attr_hash
+    annotation_hash
   end
 
   def zip_new(key, values, insitu: :lax)
@@ -51,7 +85,7 @@ module TSV
   def each(*args, &block)
     if block_given?
       super(*args) do |k,v|
-        NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
+        NamedArray.setup(v, @fields) unless @unnamed || @type == :flat || ! (Array === v)
         block.call(k, v)
       end
     else
@@ -115,8 +149,16 @@ Example:
     [@key_field] + @fields
   end
 
+  def self.all_fields(file)
+    if file.respond_to?(:all_fields)
+      file.all_fields
+    else
+      TSV.parse_header(file)["all_fields"]
+    end
+  end
+
   def options
-    self.extension_attr_hash
+    self.annotation_hash
   end
 
   def fingerprint
@@ -130,4 +172,9 @@ Example:
   def inspect
     fingerprint
   end
+
+  def merge(other)
+    self.annotate(super(other))
+  end
+
 end

data/lib/scout/tsv.rb

@@ -1,9 +1,9 @@
-require 'scout/meta_extension'
+require 'scout/annotation'
 require_relative 'tsv/util'
 require_relative 'tsv/parser'
 require_relative 'tsv/dumper'
 require_relative 'tsv/transformer'
-require_relative 'tsv/persist'
+require_relative 'persist/tsv'
 require_relative 'tsv/index'
 require_relative 'tsv/path'
 require_relative 'tsv/traverse'
@@ -11,10 +11,13 @@ require_relative 'tsv/open'
 require_relative 'tsv/attach'
 require_relative 'tsv/change_id'
 require_relative 'tsv/stream'
+require_relative 'tsv/entity'
+require_relative 'tsv/annotation'
+require_relative 'tsv/csv'
 
 module TSV
-  extend MetaExtension
-  extension_attr :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers
+  extend Annotation
+  annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :entity_options, :serializer
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -59,6 +62,10 @@ module TSV
         end
         original_setup(obj, *rest, &block)
       end
+
+      obj.save_annotation_hash if obj.respond_to?(:save_annotation_hash)
+
+      obj
     end
   end
 
@@ -68,35 +75,37 @@ module TSV
   end
 
   def self.open(file, options = {})
-    grep, invert_grep = IndiferentHash.process_options options, :grep, :invert_grep, :persist => false
+    grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
 
     persist_options = IndiferentHash.pull_keys options, :persist
-    persist_options = IndiferentHash.add_defaults persist_options, :prefix => "TSV", :type => :HDB
+    persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
+    persist_options[:data] ||= options[:data]
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
 
-    source_name, other_options = case file
-                  when StringIO
-                    [file.inspect, options]
-                  when TSV::Parser
-                    [file.options[:filename], file.options]
-                  else
-                    [file, options]
-                  end
-
-    Persist.persist(source_name, persist_options[:type], persist_options.merge(:other_options => other_options)) do |filename|
-      if filename
-        data = case persist_options[:type]
-               when :HDB, :BDB
-                 ScoutCabinet.open(filename, true, persist_options[:type])
-               when :tkh, :tkt, :tks
-                 ScoutTKRZW.open(filename, true, persist_options[:type])
-               end
+    source_name, options = 
+      case file
+      when StringIO
+        [file.inspect, options]
+      when TSV::Parser
+        [file.options[:filename], file.options]
       else
-        data = nil
+        [file, options]
       end
+
+    Persist.tsv(source_name, options, persist_options: persist_options) do |data|
       options[:data] = data if data
-      options[:filename] = TSV::Parser === file ? file.options[:filename] : file
+      options[:filename] ||= if TSV::Parser === file
+                             file.options[:filename]
+                           elsif Path === file
+                             file
+                           elsif file.respond_to?(:filename)
+                             file.filename
+                           elsif Path.is_filename?(file)
+                             file
+                           else
+                             nil
+                           end
 
       if data
         Log.debug "TSV open #{Log.fingerprint file} into #{Log.fingerprint data}"
@@ -104,14 +113,23 @@ module TSV
         Log.debug "TSV open #{Log.fingerprint file}"
       end
 
-      if TSV::Parser === file
+      tsv = if TSV::Parser === file
         TSV.parse(file, **options)
       else
-        Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
+        options[:tsv_invert_grep] ||= invert_grep if invert_grep
+        Open.open(file, grep: grep, invert_grep: invert_grep, nocache: nocache) do |f|
           TSV.parse(f, **options)
         end
       end
+
+      tsv.entity_options = entity_options
+
+      tsv
     end
   end
+
+  def to_hash
+    self.dup
+  end
 end
 

data/lib/scout/work_queue/socket.rb

@@ -17,6 +17,10 @@ class WorkQueue
       ScoutSemaphore.create_semaphore(@read_sem,1)
     end
 
+    def socket_id
+      @key
+    end
+
     def clean
       @cleaned = true
       @sread.close unless @sread.closed?
@@ -37,6 +41,10 @@ class WorkQueue
       when nil
         size_head = [0,"N"].pack 'La'
         str = size_head
+      when Annotation::AnnotatedObject
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
       when String
         payload = obj
         size_head = [payload.bytesize,"C"].pack 'La'

data/lib/scout/work_queue/worker.rb

@@ -1,13 +1,24 @@
 class WorkQueue
   class Worker
-    attr_accessor :pid, :ignore_ouput
+    attr_accessor :pid, :ignore_ouput, :queue_id
     def initialize(ignore_ouput = false)
       @ignore_output = ignore_ouput
     end
 
+    def worker_short_id
+      [object_id, pid].compact * "@"
+    end
+
+    def worker_id
+      [worker_short_id, queue_id] * "->"
+    end
+
     def run
       @pid = Process.fork do
-        Log.debug "Worker start with #{Process.pid}"
+        Signal.trap("INT") do
+          Kernel.exit! -1
+        end
+        Log.low "Worker start #{worker_id}"
         yield
       end
     end
@@ -15,6 +26,12 @@ class WorkQueue
     def process(input, output = nil, &block)
       run do
         begin
+          if output
+            Open.purge_pipes(output.swrite)
+          else
+            Open.purge_pipes
+          end
+
           while obj = input.read
             if DoneProcessing === obj
               output.write DoneProcessing.new
@@ -36,15 +53,15 @@ class WorkQueue
 
     def abort
       begin
-        Log.debug "Aborting worker #{@pid}"
-        Process.kill "INT", @pid 
+        Log.medium "Aborting worker #{worker_id}"
+        Process.kill "INT", @pid
       rescue Errno::ECHILD 
       rescue Errno::ESRCH
       end
     end
 
     def join
-      Log.debug "Joining worker #{@pid}"
+      Log.low "Joining worker #{worker_id}"
       Process.waitpid @pid
     end
 

data/lib/scout/work_queue.rb

@@ -6,18 +6,29 @@ require 'timeout'
 class WorkQueue
   attr_accessor :workers, :worker_proc, :callback
 
+  def new_worker
+    worker = Worker.new
+    worker.queue_id = queue_id
+    worker
+  end
+
   def initialize(workers = 0, &block)
     workers = workers.to_i if String === workers
     @input = WorkQueue::Socket.new
     @output = WorkQueue::Socket.new
-    @workers = workers.times.collect{ Worker.new }
+    @workers = workers.times.collect{ new_worker }
     @worker_proc = block
     @worker_mutex = Mutex.new
     @removed_workers = []
+    Log.medium "Starting queue #{queue_id} with workers: #{Log.fingerprint @workers.collect{|w| w.worker_short_id }} and sockets #{@input.socket_id} and #{@output.socket_id}"
+  end
+
+  def queue_id
+    [object_id, Process.pid] * "@"
   end
 
   def add_worker(&block)
-    worker = Worker.new
+    worker = new_worker
     @worker_mutex.synchronize do
       @workers.push(worker)
       if block_given?
@@ -41,9 +52,11 @@ class WorkQueue
     @worker_mutex.synchronize do
       worker = @workers.index{|w| w.pid == pid}
       if worker
-        Log.low "Removed worker #{pid}"
         @workers.delete_at(worker)
         @removed_workers << pid
+        Log.low "Removed worker #{pid} from #{queue_id}"
+      else
+        Log.medium "Worker #{pid} not from #{queue_id}"
       end
     end
   end
@@ -56,14 +69,14 @@ class WorkQueue
     @reader = Thread.new(Thread.current) do |parent|
       begin
         Thread.current.report_on_exception = false
-        Thread.current["name"] = "Output reader #{Process.pid}"
+        Thread.current["name"] = "Output reader #{queue_id}"
         @done_workers ||= []
         while true
           obj = @output.read
           if DoneProcessing === obj
 
             done = @worker_mutex.synchronize do
-              Log.low "Worker #{obj.pid} done"
+              Log.low "Worker #{obj.pid} from #{queue_id} done"
               @done_workers << obj.pid
               @closed && @done_workers.length == @removed_workers.length + @workers.length
             end
@@ -78,12 +91,12 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
+        Log.error "Exception in worker #{obj.pid} in queue #{queue_id}: #{obj.worker_exception.message}"
         self.abort
         @input.abort obj.worker_exception
         raise obj.worker_exception
       rescue
-        Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
+        Log.error "Exception processing output in queue #{queue_id}: #{$!.message}"
         self.abort
         raise $!
       end
@@ -95,25 +108,19 @@ class WorkQueue
 
     @waiter = Thread.new do
       Thread.current.report_on_exception = false
-      Thread.current["name"] = "Worker waiter #{Process.pid}"
+      Thread.current["name"] = "Worker waiter #{queue_id}"
       while true
         break if @worker_mutex.synchronize{ @workers.empty? }
-        begin
-          Timeout.timeout(1) do
-            begin
-              pid, status = Process.wait2
-              remove_worker(pid) if pid
-            rescue Exception
-              Log.exception $!
-            end
-          end
-        rescue Timeout::Error
-          pids = @worker_mutex.synchronize{ @workers.collect{|w| w.pid } }
-          pids.each do |p|
-            pid, status = Process.wait2 p, Process::WNOHANG
+        threads = @workers.collect do |w|
+          t = Thread.new do
+            Thread.current["name"] = "Worker waiter #{queue_id} worker #{w.pid}"
+            pid, status = Process.wait2 w.pid
             remove_worker(pid) if pid
           end
+          Thread.pass until t["name"]
+          t
         end
+        threads.each do |t| t.join end
       end
     end
 
@@ -131,16 +138,23 @@ class WorkQueue
   end
 
   def abort
-    Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
+    Log.low "Aborting #{@workers.length} workers in queue #{queue_id}"
     @worker_mutex.synchronize do
-      @workers.each{|w| w.abort }
+      @workers.each do |w| 
+        ScoutSemaphore.post_semaphore(@output.write_sem) if @output
+        ScoutSemaphore.post_semaphore(@input.read_sem) if @input
+        w.abort 
+      end
     end
   end
 
   def close
     @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
-      @input.write DoneProcessing.new() unless @input.closed_write?
+      begin
+        @input.write DoneProcessing.new() unless @input.closed_write?
+      rescue IOError
+      end
     end
   end