scout-gear 7.3.0 → 8.1.0

data/lib/scout/tsv/dumper.rb

@@ -1,38 +1,14 @@
 module TSV
   class Dumper
-    def self.header_lines(key_field, fields, entry_hash = nil)
-      if Hash === entry_hash 
-        sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
-        preamble = entry_hash[:preamble]
-        header_hash = entry_hash[:header_hash]
-      end
-
-      header_hash = "#" if header_hash.nil?
-
-      preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
-
-      str = "" 
-      str << preamble.strip << "\n" if preamble and not preamble.empty?
-      if fields
-        if fields.empty?
-          str << header_hash << (key_field || "ID").to_s << "\n" 
-        else
-          str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
-        end
-      end
-
-      str
-    end
-
     def self.header(options={})
-      key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options, 
-        :key_field, :fields, :sep, :header_hash, :preamble,
+      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options, 
+        :key_field, :fields, :sep, :header_hash, :preamble, :unnamed,
         :sep => "\t", :header_hash => "#", :preamble => true
 
-      if fields.nil? || key_field.nil?
+      if fields.nil?
         fields_str = nil
       else
-        fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
+        fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
       end
 
       if preamble && options.values.compact.any?
@@ -45,31 +21,64 @@ module TSV
     end
 
 
-    attr_accessor :options
+    attr_accessor :options, :initialized, :type, :sep
     def initialize(options = {})
+      options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
       @sep, @type = IndiferentHash.process_options options, 
         :sep, :type, 
         :sep => "\t", :type => :double
       @options = options
+      @options[:type] = @type
       @sout, @sin = Open.pipe
+      Log.low{"Dumper pipe #{[Log.fingerprint(@sin), Log.fingerprint(@sout)] * " -> "}"}
+      @initialized = false
+      @mutex = Mutex.new
       ConcurrentStream.setup(@sin, pair: @sout)
       ConcurrentStream.setup(@sout, pair: @sin)
     end
 
-    def init
-      header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
-      @sin.puts header if header and ! header.empty?
+    def key_field
+      @options[:key_field]
+    end
+    
+    def fields
+      @options[:fields]
     end
 
-    def add(key, value)
+    def key_field=(key_field)
+      @options[:key_field] = key_field
+    end
+    
+    def fields=(fields)
+      @options[:fields] = fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
 
-      case @type
-      when :single
-        @sin.puts key + @sep + value
-      when :list, :flat
-        @sin.puts key + @sep + value * @sep
-      when :double
-        @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
+
+    def init(preamble: true)
+      header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
+      @mutex.synchronize do
+        @initialized = true
+        @sin.puts header if header and ! header.empty?
+      end
+    end
+
+    def add(key, value)
+      @mutex.synchronize do
+
+        key = key.to_s unless String === key
+        case @type
+        when :single
+          @sin.puts key + @sep + value.to_s
+        when :list, :flat
+          @sin.puts key + @sep + value * @sep
+        when :double
+          @sin.puts key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep
+        end
       end
     end
 
@@ -85,15 +94,32 @@ module TSV
     def abort(exception=nil)
       @sin.abort(exception)
     end
+
+    def tsv(*args)
+      TSV.open(stream, *args)
+    end
+
+    def fingerprint
+      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+    end
+
+    def digest_str
+      fingerprint
+    end
+
+    def inspect
+      fingerprint
+    end
   end
 
-  def stream
-    dumper = TSV::Dumper.new self.extension_attr_hash
-    dumper.init
+  def dumper_stream(options = {})
+    preamble = IndiferentHash.process_options options, :preamble, :preamble => true
+    dumper = TSV::Dumper.new self.extension_attr_hash.merge(options)
     t = Thread.new do 
       begin
         Thread.current.report_on_exception = true
         Thread.current["name"] = "Dumper thread"
+        dumper.init(preamble: preamble)
         self.each do |k,v|
           dumper.add k, v
         end
@@ -103,10 +129,12 @@ module TSV
       end
     end
     Thread.pass until t["name"]
-    dumper.stream
+    s = dumper.stream
+    ConcurrentStream.setup(s, :threads => [t])
+    s
   end
 
-  def to_s
-    stream.read
+  def to_s(options = {})
+    dumper_stream(options).read
   end
 end

data/lib/scout/tsv/index.rb

@@ -1,13 +1,16 @@
 require_relative 'parser'
+require_relative 'transformer'
 require_relative 'persist/fix_width_table'
 module TSV
-  def self.index(tsv_file, target: 0, fields: nil, order: true, **kwargs)
-    persist, type = IndiferentHash.process_options kwargs,
-      :persist, :persist_type,
+  def self.index(tsv_file, target: 0, fields: nil, order: true, bar: nil, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => "HDB"
     kwargs.delete :type
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+    fields = :all if fields.nil?
+
+    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, :prefix => "Index", :other_options => kwargs)) do |filename|
       if filename
         index = ScoutCabinet.open(filename, true, type)
         TSV.setup(index, :type => :single)
@@ -16,11 +19,19 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      dummy_data = TSV.setup({}, :key_field => "Key", :fields => ["Target"])
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
+
       if order
         tmp_index = {}
-        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :double, into: dummy_data, unnamed: true, **kwargs do |k,values|
+        include_self = fields == :all || (Array === fields) && fields.include?(target)
+        target_key_field, source_field_names = Open.traverse tsv_file, type: :double, key_field: target, fields: fields, unnamed: true, bar: bar, **kwargs do |k,values|
+          tmp_index[k] ||= [[k]] if include_self
           values.each_with_index do |list,i|
+            i += 1 if include_self
             list.each do |e|
               tmp_index[e] ||= []
               tmp_index[e][i] ||= []
@@ -31,16 +42,24 @@ module TSV
         tmp_index.each do |e,list|
           index[e] = list.flatten.compact.uniq.first
         end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
+
+        tmp_index = {}
+
       else
-        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :flat, into: dummy_data, unnamed: true, **kwargs do |k,values|
+        target_key_field, source_field_names =  Open.traverse tsv_file, key_field: target, fields: fields, type: :flat, unnamed: true, bar: bar, **kwargs do |k,values|
           values.each do |e|
             index[e] = k unless index.include?(e)
           end
         end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
       end
 
-      index.key_field = dummy_data.fields * ", "
-      index.fields = [dummy_data.key_field]
+
       index
     end
   end
@@ -49,17 +68,26 @@ module TSV
     TSV.index(self, *args, **kwargs, &block)
   end
 
-  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
-    persist, type = IndiferentHash.process_options kwargs,
-      :persist, :persist_type,
+  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, bar: nil, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => :fwt
     kwargs.delete :type
+    kwargs[:unnamed] = true
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+    Persist.persist(tsv_file, type, 
+                    :persist => persist, :prefix => "RangeIndex[#{[start_field, end_field]*"-"}]", update: persist_update,
+                    :other_options => kwargs) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{[start_field, end_field]*"-"}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
 
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field] do |key, values|
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], bar: bar, **kwargs do |key, values|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
@@ -81,81 +109,51 @@ module TSV
     end
   end
 
+  def self.pos_index(tsv_file, pos_field = nil, key_field: :key, bar: nil, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+    kwargs[:unnamed] = true
+
+    Persist.persist(tsv_file, type, 
+                    :persist => persist, :prefix => "RangeIndex[#{pos_field}]", update: persist_update,
+                    :other_options => kwargs) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{pos_field}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, bar: bar, **kwargs do |key, pos|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        if Array === pos
+          pos.zip(end_pos).each do |p|
+            index_pos << [key, p]
+          end
+        else
+          index_data << [key, pos]
+        end
+      end
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, false)
+      index.add_point index_data
+      index.read
+      index
+    end
+  end
+
   def range_index(*args, **kwargs, &block)
     TSV.range_index(self, *args, **kwargs, &block)
   end
 
-
-  #def range_index(start_field = nil, end_field = nil, options = {})
-  #  start_field ||= "Start"
-  #  end_field ||= "End"
-
-  #  options = Misc.add_defaults options,
-  #    :persist => false, :persist_file => nil, :persist_update => false 
-
-  #  persist_options = Misc.pull_keys options, :persist
-  #  persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
-
-  #  Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do 
-  #    max_key_size = 0
-  #    index_data = []
-  #    with_unnamed do
-  #      with_monitor :desc => "Creating Index Data", :step => 10000 do
-  #        through :key, [start_field, end_field] do |key, values|
-  #          key_size = key.length
-  #          max_key_size = key_size if key_size > max_key_size
-
-  #          start_pos, end_pos = values
-  #          if Array === start_pos
-  #            start_pos.zip(end_pos).each do |s,e|
-  #              index_data << [key, [s.to_i, e.to_i]]
-  #            end
-  #          else
-  #            index_data << [key, [start_pos.to_i, end_pos.to_i]]
-  #          end
-  #        end
-  #      end
-  #    end
-
-  #    index = FixWidthTable.get(:memory, max_key_size, true)
-  #    index.add_range index_data
-  #    index.read
-  #    index
-  #  end
-  #end
-
-  #def self.range_index(file, start_field = nil, end_field = nil, options = {})
-  #  start_field ||= "Start"
-  #  end_field ||= "End"
-
-  #  data_options = Misc.pull_keys options, :data
-  #  filename = case
-  #             when (String === file or Path === file)
-  #               file
-  #             when file.respond_to?(:filename)
-  #               file.filename
-  #             else
-  #               file.object_id.to_s
-  #             end
-  #  persist_options = Misc.pull_keys options, :persist
-  #  persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
-
-  #  filters = Misc.process_options options, :filters
-
-  #  if filters
-  #    filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
-  #  end
-
-  #  Persist.persist(filename, :fwt, persist_options) do
-  #    tsv = TSV.open(file, data_options)
-  #    if filters
-  #      tsv.filter
-  #      filters.each do |match, value|
-  #        tsv.add_filter match, value
-  #      end
-  #    end
- 
-  #    tsv.range_index(start_field, end_field, options)
-  #  end
-  #end
+  def pos_index(*args, **kwargs, &block)
+    TSV.pos_index(self, *args, **kwargs, &block)
+  end
 end

data/lib/scout/tsv/open.rb

@@ -1,61 +1,56 @@
 require_relative '../open'
+require_relative '../work_queue'
+
+module MultipleResult
+  def self.setup(obj)
+    obj.extend MultipleResult
+    obj
+  end
+end
+
 module Open
   def self.traverse_add(into, res)
-    case into
-    when TSV::Dumper
-      into.add *res
-    when TSV, Hash
-      key, value = res
-      into[key] = value
+    if Array === res && MultipleResult === res
+      res.each do |_res|
+        traverse_add into, _res
+      end
+    else
+      case into
+      when defined?(TSV::Dumper) && TSV::Dumper
+        into.add *res
+      when TSV, Hash
+        key, value = res
+        if into.type == :double
+          into.zip_new key, value, insitu: false
+        else
+          into[key] = value
+        end
+      when Array, Set
+        into << res
+      when IO, StringIO
+        into.puts res
+      end
     end
   end
 
-  #def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
-  #  case obj
-  #  when TSV
-  #    obj.traverse options[:key_field], options[:fields], **options do |k,v|
-  #      res = yield k, v
-  #    end
-  #  when String
-  #    f = Open.open(obj)
-  #    self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
-  #  when Step
-  #    self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
-  #  when IO
-  #    if into && (IO === into || into.respond_to?(:stream) )
-  #      into_thread = Thread.new do 
-  #        Thread.current.report_on_exception = false
-  #        Thread.current["name"] = "Traverse into"
-  #        TSV.parse obj, **options do |k,v|
-  #          begin
-  #            res = block.call k, v
-  #            traverse_add into, res
-  #          rescue
-  #            into.abort $!
-  #          end
-  #          nil
-  #        end
-  #        into.close if into.respond_to?(:close)
-  #      end
-  #      Thread.pass until into_thread
-  #      into
-  #    else
-  #      TSV.parse obj, **options do |k,v|
-  #        block.call k, v
-  #        nil
-  #      end
-  #    end
-  #  end
-  #end
-
-  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, **options, &block)
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, keep_open: false, **options, &block)
+    cpus = nil if cpus.to_i == 1
+
+    if into == :stream
+      sout, sin = Open.pipe
+      ConcurrentStream.setup(sout, :pair => sin)
+      ConcurrentStream.setup(sin, :pair => sout)
+      self.traverse(obj, into: sin, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
+      return sout
+    end
 
     if into || bar
       orig_callback = callback if callback
-      bar = Log::ProgressBar.get_obj_bar(bar, obj)
+      bar = Log::ProgressBar.get_obj_bar(obj, bar) if bar
+      bar.init if bar
       callback = proc do |res|
         bar.tick if bar
-        traverse_add into, res if into
+        traverse_add into, res if into && ! res.nil?
         orig_callback.call res if orig_callback
       end
 
@@ -65,59 +60,115 @@ module Open
           Thread.current["name"] = "Traverse into"
           error = false
           begin
-            self.traverse(obj, callback: callback, **options, &block)
-            into.close if into.respond_to?(:close)
+            self.traverse(obj, callback: callback, cpus: cpus, unnamed: unnamed, **options, &block)
+            into.close if ! keep_open && into.respond_to?(:close)
             bar.remove if bar
           rescue Exception
             into.abort($!) if into.respond_to?(:abort)
             bar.remove($!) if bar
           end
         end
-        Thread.pass until into_thread
+
+        Thread.pass until into_thread["name"]
+
+        case into
+        when IO
+          ConcurrentStream.setup into, :threads => into_thread
+        when TSV::Dumper
+          ConcurrentStream.setup into.stream, :threads => into_thread
+        end
         return into
       end
     end
 
-    begin
-      case obj
-      when TSV
-        obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v|
-          res = block.call(k, v)
-          callback.call res if callback
-          nil
-        end
-      when Array
-        obj.each do |line|
-          res = block.call(line)
-          callback.call res if callback
-          nil
-        end
-      when String
-        f = Open.open(obj)
-        self.traverse(f, cpus: cpus, callback: callback, **options, &block)
-      when Step
-        raise obj.exception if obj.error?
-        self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
-      when IO
-        TSV.parse obj, **options do |k,v|
-          res = block.call k, v
-          callback.call res if callback
-          nil
-        end
-      else
-        TSV.parse obj, **options do |k,v|
-          res = block.call k, v
-          callback.call res if callback
-          nil
+    if cpus
+      queue = WorkQueue.new cpus do |args|
+        block.call *args
+      end
+
+      queue.process do |res|
+        callback.call res
+      end
+      
+      begin
+        self.traverse(obj, **options) do |*args|
+          queue.write args
         end
+
+        queue.close
+
+        queue.join(false)
+
+        bar.remove if bar
+        return into
+      rescue Exception
+        bar.remove($!) if bar
+        queue.abort
+        raise $!
+      ensure
+        queue.clean
       end
+    end
+
+    begin
+      res = case obj
+            when TSV
+              obj.traverse  unnamed: unnamed, **options do |k,v,f|
+                res = block.call(k, v, f)
+                callback.call res if callback
+                nil
+              end
+            when Array
+              obj.each do |line|
+                res = block.call(line)
+                callback.call res if callback
+                nil
+              end
+            when String
+              obj = obj.produce_and_find if Path === obj
+              f = Open.open(obj)
+              self.traverse(f, cpus: cpus, callback: callback, **options, &block)
+            when Step
+              raise obj.exception if obj.error?
+              self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
+            when IO
+              if options[:type] == :array || options[:type] == :line
+                Log.low "Traverse stream by lines #{Log.fingerprint obj}"
+                while line = obj.gets
+                  line.strip!
+                  res = block.call(line)
+                  callback.call res if callback
+                end
+              else
+                Log.low "Traverse stream with parser #{Log.fingerprint obj}"
+                parser = TSV::Parser.new obj
+                parser.traverse **options do |k,v,f|
+                  res = block.call k,v,f
+                  callback.call res if callback
+                  nil
+                end
+              end
+            when TSV::Parser
+              obj.traverse **options do |k,v,f|
+                res = block.call k, v, f
+                callback.call res if callback
+                nil
+              end
+            else
+              TSV.parse obj, **options do |k,v|
+                res = block.call k, v
+                callback.call res if callback
+                nil
+              end
+            end
       bar.remove if bar
-    rescue
-      bar.abort($!) if bar
-      raise $!
+    rescue Exception => exception
+      exception = obj.stream_exception if (ConcurrentStream === obj) && obj.stream_exception
+      bar.error if bar
+      raise exception
     end
 
-    into
+    into || res
   end
 end
 
@@ -125,4 +176,28 @@ module TSV
   def self.traverse(*args, **kwargs, &block)
     Open.traverse(*args, **kwargs, &block)
   end
+
+  def self.process_stream(stream, header_hash: "#", &block)
+    sout = Open.open_pipe do |sin|
+      while line = stream.gets 
+        break unless line.start_with?(header_hash)
+        sin.puts line
+      end
+      yield sin, line
+    end
+  end
+
+  def self.collapse_stream(stream, *args, **kwargs, &block)
+    stream = stream.stream if stream.respond_to?(:stream)
+    self.process_stream(stream) do |sin, line|
+      collapsed = Open.collapse_stream(stream, line: line)
+      Open.consume_stream(collapsed, false, sin)
+    end
+  end
+
+  def collapse_stream(*args, **kwargs, &block)
+    TSV.collapse_stream(self.dumper_stream, *args, **kwargs, &block)
+  end
+
+
 end