scout-gear 10.4.0 → 10.7.0

data/lib/scout/tsv/dumper.rb

@@ -7,6 +7,8 @@ module TSV
 
       if fields.nil?
         fields_str = nil
+      elsif fields.empty?
+        fields_str = "#{header_hash}#{key_field || "Id"}"
       else
         fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
       end
@@ -14,7 +16,7 @@ module TSV
       if String === preamble
         preamble_str = preamble
       elsif preamble && options.values.compact.any?
-        preamble_str = "#: " << IndiferentHash.hash2string(options)
+        preamble_str = "#: " << IndiferentHash.hash2string(options.merge(serializer: nil))
       else
         preamble_str = nil
       end
@@ -24,7 +26,7 @@ module TSV
     end
 
 
-    attr_accessor :options, :initialized, :type, :sep
+    attr_accessor :options, :initialized, :type, :sep, :filename, :namespace
     def initialize(options = {})
       options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
       @sep, @type = IndiferentHash.process_options options, 
@@ -35,11 +37,18 @@ module TSV
       @sout, @sin = Open.pipe
       Log.low{"Dumper pipe #{[Log.fingerprint(@sin), Log.fingerprint(@sout)] * " -> "}"}
       @initialized = false
+      @filename = options[:filename]
       @mutex = Mutex.new
       ConcurrentStream.setup(@sin, pair: @sout)
       ConcurrentStream.setup(@sout, pair: @sin)
     end
 
+    def set_stream(stream)
+      @sin.close
+      @sout.close
+      @sout = @sin = stream
+    end
+
     def key_field
       @options[:key_field]
     end
@@ -56,6 +65,10 @@ module TSV
       @options[:fields] = fields
     end
 
+    def namespace=(namespace)
+      @options[:namespace] = namespace
+    end
+
     def all_fields
       return nil if fields.nil?
       [key_field] + fields
@@ -66,7 +79,7 @@ module TSV
       header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
       @mutex.synchronize do
         @initialized = true
-        @sin.puts header if header and ! header.empty?
+        @sin << header << "\n" if header and ! header.empty?
       end
     end
 
@@ -74,20 +87,36 @@ module TSV
       @mutex.synchronize do
 
         key = key.to_s unless String === key
-        case @type
-        when :single
-          @sin.puts key + @sep + value.to_s
-        when :list, :flat
-          @sin.puts key + @sep + value * @sep
-        when :double
-          @sin.puts key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep
+        if value.nil? || (Array === value && value.empty?)
+          @sin << key << "\n"
+        else
+          case @type
+          when :single
+            @sin << key + @sep + value.to_s << "\n"
+          when :list, :flat
+            @sin << key + @sep + value * @sep << "\n"
+          when :double
+            @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
+          else
+            if Array === value 
+              if Array === value.first
+                @sin << key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep << "\n"
+              else
+                @sin << key + @sep + value * @sep << "\n"
+              end
+            else
+              @sin << key + @sep + value.to_s << "\n"
+            end
+          end
         end
       end
     end
 
     def close
-      @sin.close
-      @sin.join
+      if @sin != @sout
+        @sin.close if @sin.respond_to?(:close) && ! @sin.closed?
+        @sin.join if @sin.respond_to?(:join) && ! @sin.joined?
+      end
     end
 
     def stream
@@ -95,7 +124,7 @@ module TSV
     end
 
     def abort(exception=nil)
-      @sin.abort(exception)
+      @sin.abort(exception) if @sin.respond_to?(:abort)
     end
 
     def tsv(*args)
@@ -116,28 +145,87 @@ module TSV
   end
 
   def dumper_stream(options = {})
-    preamble = IndiferentHash.process_options options, :preamble, :preamble => true
-    dumper = TSV::Dumper.new self.extension_attr_hash.merge(options)
-    t = Thread.new do 
+    preamble, unmerge, keys, stream = IndiferentHash.process_options options, 
+      :preamble, :unmerge, :keys, :stream,
+      :preamble => true, :unmerge => false
+    unmerge = false unless @type === :double
+    dumper = TSV::Dumper.new self.annotation_hash.merge(options)
+
+    dump_entry = Proc.new do |k,value_list|
+      if unmerge
+        max = value_list.collect{|v| v.length}.max
+
+        if unmerge == :expand and max > 1
+          value_list = value_list.collect do |values|
+            if values.length == 1
+              [values.first] * max
+            else
+              values
+            end
+          end
+        end
+
+        NamedArray.zip_fields(value_list).each do |values|
+          dumper.add k, values
+        end
+      else
+        dumper.add k, value_list
+      end
+    end
+
+    if stream.nil?
+      t = Thread.new do 
+        begin
+          Thread.current.report_on_exception = true
+          Thread.current["name"] = "Dumper thread"
+          dumper.init(preamble: preamble)
+
+          if keys
+            keys.each do |k|
+              dump_entry.call k, self[k]
+            end
+          else
+            self.each &dump_entry
+          end
+
+          dumper.close
+        rescue
+          dumper.abort($!)
+        end
+      end
+      Thread.pass until t["name"]
+      stream = dumper.stream
+      ConcurrentStream.setup(stream, :threads => [t])
+      stream
+    else
+      dumper.set_stream stream
       begin
-        Thread.current.report_on_exception = true
-        Thread.current["name"] = "Dumper thread"
         dumper.init(preamble: preamble)
-        self.each do |k,v|
-          dumper.add k, v
+        if keys
+          keys.each do |k|
+            dump_entry.call k, self[k]
+          end
+        else
+          self.each &dump_entry
         end
+
         dumper.close
       rescue
         dumper.abort($!)
       end
+      stream
     end
-    Thread.pass until t["name"]
-    s = dumper.stream
-    ConcurrentStream.setup(s, :threads => [t])
-    s
   end
 
   def to_s(options = {})
-    dumper_stream(options).read
+    dumper_stream({stream: ''}.merge(options))
+  end
+
+  alias stream dumper_stream
+
+  def write_file(file)
+    Open.open(file, mode: 'w') do |f|
+      dumper_stream(stream: f)
+    end
   end
 end

data/lib/scout/tsv/index.rb

@@ -1,25 +1,72 @@
 require_relative 'parser'
 require_relative 'transformer'
-require_relative 'persist/fix_width_table'
+require_relative '../persist/tsv'
 module TSV
+
+  def self.select_prefix_str(select)
+    str = begin
+            case select
+            when nil
+              nil
+            when Array
+              case select.first
+              when nil
+                nil
+              when Array
+                select.collect{|p| p * "="}*","
+              else
+                select.collect{|p| p.to_s }*"="
+              end
+            when Hash
+              if select.empty?
+                nil
+              else
+                select.collect do |key,value|
+                  [key.to_s, value.to_s] * "="
+                end * ","
+              end
+            end
+          rescue
+            Log.warn "Error in select_prefix_str: #{Log.fingerprint(select)}: #{$!.message}"
+            str = nil
+          end
+    if str.nil?
+      ""
+    else
+      "[select:#{str}]"
+    end
+  end
+
   def self.index(tsv_file, target: :key, fields: nil, order: true, bar: nil, **kwargs)
-    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
-      :persist, :persist_type, :persist_update, :data_persist,
-      :persist => false, :persist_type => "HDB"
-    kwargs.delete :type
+    kwargs = IndiferentHash.add_defaults kwargs, unnamed: true
+    engine = IndiferentHash.process_options kwargs, :engine
 
     fields = :all if fields.nil?
 
-    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, prefix: "Index", other_options: {fields: fields, target: target, order: order})) do |filename|
+    prefix = case fields
+             when :all
+               "Index[#{target}]"
+             else
+               "Index[#{Log.fingerprint(fields)}->#{target}]"
+             end
+
+    prefix += select_prefix_str(kwargs[:select])
+
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+    persist_options = IndiferentHash.add_defaults persist_options, :prefix => prefix, :engine => :HDB, :persist => false
+
+    data_options = IndiferentHash.pull_keys kwargs, :data
+
+    Persist.persist(tsv_file, persist_options[:engine], persist_options.merge(other_options: kwargs.merge(target: target, fields: fields, order: order, data_options: data_options))) do |filename|
       if filename
-        index = ScoutCabinet.open(filename, true, type)
+        index = ScoutCabinet.open(filename, true, engine)
         TSV.setup(index, :type => :single)
         index.extend TSVAdapter 
       else
         index = TSV.setup({}, :type => :single)
       end
 
-      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+      tsv_file = TSV.open(tsv_file, **data_options) if ! TSV === tsv_file
 
       log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
       Log.low log_msg
@@ -28,7 +75,7 @@ module TSV
       if order
         tmp_index = {}
         include_self = fields == :all || (Array === fields) && fields.include?(target)
-        target_key_field, source_field_names = Open.traverse tsv_file, type: :double, key_field: target, fields: fields, unnamed: true, bar: bar, **kwargs do |k,values|
+        target_key_field, source_field_names = Open.traverse tsv_file, type: :double, key_field: target, fields: fields, bar: bar, **kwargs do |k,values|
           tmp_index[k] ||= [[k]] if include_self
           values.each_with_index do |list,i|
             i += 1 if include_self
@@ -54,11 +101,10 @@ module TSV
             index[e] = k unless index.include?(e)
           end
         end
-
-        index.key_field = source_field_names * ","
-        index.fields = [target_key_field]
       end
 
+      index.key_field = source_field_names * ","
+      index.fields = [target_key_field]
 
       index
     end
@@ -66,20 +112,23 @@ module TSV
 
   def index(*args, **kwargs, &block)
     TSV.index(self, *args, **kwargs, &block)
-  end
+ end
 
   def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, bar: nil, **kwargs)
-    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
-      :persist, :persist_type, :persist_update, :data_persist,
-      :persist => false, :persist_type => :fwt
-    kwargs.delete :type
-    kwargs[:unnamed] = true
+    kwargs = IndiferentHash.add_defaults kwargs, unnamed: true
+    type, data_persist = IndiferentHash.process_options kwargs, :type, :data_persist
+
+    prefix = "RangeIndex[#{start_field}-#{end_field}]"
 
-    Persist.persist(tsv_file, type, 
-                    :persist => persist, :prefix => "RangeIndex[#{[start_field, end_field]*"-"}]", update: persist_update,
-                    :other_options => kwargs) do |filename|
+    prefix += select_prefix_str(kwargs[:select])
 
-      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+    persist_options = IndiferentHash.add_defaults persist_options, :prefix => prefix, :type => :fwt, :persist => true
+
+    data_options = IndiferentHash.pull_keys kwargs, :data
+
+    Persist.persist(tsv_file, persist_options[:type], persist_options.merge(other_options: kwargs.merge(start_field: start_field, end_field: end_field, key_field: key_field))) do |filename|
+      tsv_file = TSV.open(tsv_file, *data_options) if data_options[:persist] && ! TSV === tsv_file
 
       log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{[start_field, end_field]*"-"}"
       Log.low log_msg
@@ -87,7 +136,7 @@ module TSV
 
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], bar: bar, **kwargs do |key, values|
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], bar: bar, unnamed: true, **kwargs do |key, values|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
@@ -110,31 +159,34 @@ module TSV
   end
 
   def self.pos_index(tsv_file, pos_field = nil, key_field: :key, bar: nil, **kwargs)
-    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
-      :persist, :persist_type, :persist_update, :data_persist,
-      :persist => false, :persist_type => :fwt
-    kwargs.delete :type
-    kwargs[:unnamed] = true
+    kwargs = IndiferentHash.add_defaults kwargs, unnamed: true
+    type, data_persist = IndiferentHash.process_options kwargs, :type
+
+    prefix = "PositionIndex[#{pos_field}]"
+
+    prefix += select_prefix_str(kwargs[:select])
+
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+    persist_options = IndiferentHash.add_defaults persist_options, :prefix => prefix, :type => :fwt, :persist => true
 
-    Persist.persist(tsv_file, type, 
-                    :persist => persist, :prefix => "RangeIndex[#{pos_field}]", update: persist_update,
-                    :other_options => kwargs) do |filename|
+    data_options = IndiferentHash.pull_keys kwargs, :data
 
-      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+    Persist.persist(tsv_file, persist_options[:type], persist_options.merge(other_options: kwargs.merge(pos_field: pos_field, key_field: key_field))) do |filename|
+      tsv_file = TSV.open(tsv_file, *data_options) if data_options[:persist] && ! TSV === tsv_file
 
-      log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{pos_field}"
+      log_msg = "PositionIndex #{Log.fingerprint tsv_file} #{pos_field}"
       Log.low log_msg
       bar = log_msg if TrueClass === bar
 
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, bar: bar, **kwargs do |key, pos|
+      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :flat, cast: :to_i, bar: bar, **kwargs do |key, pos|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
         if Array === pos
-          pos.zip(end_pos).each do |p|
-            index_pos << [key, p]
+          pos.each do |p|
+            index_data << [key, p]
           end
         else
           index_data << [key, pos]

data/lib/scout/tsv/open.rb

@@ -42,6 +42,11 @@ module Open
       ConcurrentStream.setup(sin, :pair => sout)
       self.traverse(obj, into: sin, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
       return sout
+    elsif Path === into
+      Open.write(into) do |io|
+        self.traverse(obj, into: io, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
+      end
+      return into
     end
 
     if into || bar
@@ -64,8 +69,8 @@ module Open
             into.close if ! keep_open && into.respond_to?(:close)
             bar.remove if bar
           rescue Exception
-            into.abort($!) if into.respond_to?(:abort)
             bar.remove($!) if bar
+            into.abort($!) if into.respond_to?(:abort)
           end
         end
 
@@ -87,7 +92,7 @@ module Open
       end
 
       queue.process do |res|
-        callback.call res
+        callback.call res if callback
       end
       
       begin
@@ -134,7 +139,7 @@ module Open
               obj = obj.produce_and_find if Path === obj
               f = Open.open(obj)
               self.traverse(f, cpus: cpus, callback: callback, **options, &block)
-            when Step
+            when (defined?(Step) && Step)
               raise obj.exception if obj.error?
               self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
             when IO
@@ -147,7 +152,7 @@ module Open
                 end
               else
                 Log.low "Traverse stream with parser #{Log.fingerprint obj}"
-                parser = TSV::Parser.new obj
+                parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])
                 parser.traverse **options do |k,v,f|
                   res = block.call k,v,f
                   callback.call res if callback
@@ -161,10 +166,18 @@ module Open
                 nil
               end
             else
-              TSV.parse obj, **options do |k,v|
-                res = block.call k, v
-                callback.call res if callback
-                nil
+              if obj.respond_to?(:pop)
+                while elem = obj.pop
+                  res = block.call elem
+                  callback.call res if callback
+                  break unless obj.any?
+                end
+              else
+                TSV.parse obj, **options do |k,v|
+                  res = block.call k, v
+                  callback.call res if callback
+                  nil
+                end
               end
             end
       bar.remove if bar