scout-gear 8.0.0 → 8.1.0

data/lib/scout/resource/util.rb

@@ -8,7 +8,6 @@ module Resource
 
     map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
     map_order -= [:current, "current"]
-    map_order << :current
 
     choices = []
     map_order.uniq.each do |name|
@@ -17,6 +16,7 @@ module Resource
       next if pattern.nil?
 
       pattern = pattern.sub('{PWD}', Dir.pwd)
+      pattern = pattern.sub('{HOME}', ENV["HOME"])
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern
           .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
@@ -35,14 +35,20 @@ module Resource
       end
     end
 
-    Path.setup(choices.sort_by{|s| s.length }.first, self, nil, path_maps)
+    identified = choices.sort_by{|s| s.length }.first
+
+    Path.setup(identified || path, self, nil, path_maps)
   end
 
-  def self.relocate(path)
-    return path if Open.exists?(path)
+  def self.identify(path)
     resource = path.pkgdir if Path === path
     resource = Scout unless Resource === resource
     unlocated = resource.identify path
+  end
+
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    unlocated = identify(path)
     unlocated.find
   end
 end

data/lib/scout/tsv/dumper.rb

@@ -28,7 +28,9 @@ module TSV
         :sep, :type, 
         :sep => "\t", :type => :double
       @options = options
+      @options[:type] = @type
       @sout, @sin = Open.pipe
+      Log.low{"Dumper pipe #{[Log.fingerprint(@sin), Log.fingerprint(@sout)] * " -> "}"}
       @initialized = false
       @mutex = Mutex.new
       ConcurrentStream.setup(@sin, pair: @sout)

data/lib/scout/tsv/index.rb

@@ -21,12 +21,14 @@ module TSV
 
       tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
 
-      bar = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}" if TrueClass === bar
+      log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
 
       if order
         tmp_index = {}
         include_self = fields == :all || (Array === fields) && fields.include?(target)
-        target_key_field, source_field_names = Open.traverse tsv_file, key_field: target, fields: fields, type: :double, unnamed: true, bar: bar, **kwargs do |k,values|
+        target_key_field, source_field_names = Open.traverse tsv_file, type: :double, key_field: target, fields: fields, unnamed: true, bar: bar, **kwargs do |k,values|
           tmp_index[k] ||= [[k]] if include_self
           values.each_with_index do |list,i|
             i += 1 if include_self
@@ -66,19 +68,26 @@ module TSV
     TSV.index(self, *args, **kwargs, &block)
   end
 
-  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
+  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, bar: nil, **kwargs)
     persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
       :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => :fwt
     kwargs.delete :type
+    kwargs[:unnamed] = true
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :prefix => "RangeIndex", :other_options => kwargs, update: persist_update)) do |filename|
+    Persist.persist(tsv_file, type, 
+                    :persist => persist, :prefix => "RangeIndex[#{[start_field, end_field]*"-"}]", update: persist_update,
+                    :other_options => kwargs) do |filename|
 
       tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
 
+      log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{[start_field, end_field]*"-"}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
+
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], **kwargs do |key, values|
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], bar: bar, **kwargs do |key, values|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
@@ -100,23 +109,26 @@ module TSV
     end
   end
 
-  def range_index(*args, **kwargs, &block)
-    TSV.range_index(self, *args, **kwargs, &block)
-  end
-
-  def self.pos_index(tsv_file, pos_field = nil, key_field: :key, **kwargs)
+  def self.pos_index(tsv_file, pos_field = nil, key_field: :key, bar: nil, **kwargs)
     persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
       :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => :fwt
     kwargs.delete :type
+    kwargs[:unnamed] = true
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, update: persist_update, :prefix => "RangeIndex", :other_options => kwargs)) do |filename|
+    Persist.persist(tsv_file, type, 
+                    :persist => persist, :prefix => "RangeIndex[#{pos_field}]", update: persist_update,
+                    :other_options => kwargs) do |filename|
 
       tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
 
+      log_msg = "RangeIndex #{Log.fingerprint tsv_file} #{pos_field}"
+      Log.low log_msg
+      bar = log_msg if TrueClass === bar
+
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, **kwargs do |key, pos|
+      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, bar: bar, **kwargs do |key, pos|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
@@ -137,81 +149,11 @@ module TSV
     end
   end
 
+  def range_index(*args, **kwargs, &block)
+    TSV.range_index(self, *args, **kwargs, &block)
+  end
+
   def pos_index(*args, **kwargs, &block)
     TSV.pos_index(self, *args, **kwargs, &block)
   end
-
-
-  #def range_index(start_field = nil, end_field = nil, options = {})
-  #  start_field ||= "Start"
-  #  end_field ||= "End"
-
-  #  options = Misc.add_defaults options,
-  #    :persist => false, :persist_file => nil, :persist_update => false 
-
-  #  persist_options = Misc.pull_keys options, :persist
-  #  persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
-
-  #  Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do 
-  #    max_key_size = 0
-  #    index_data = []
-  #    with_unnamed do
-  #      with_monitor :desc => "Creating Index Data", :step => 10000 do
-  #        through :key, [start_field, end_field] do |key, values|
-  #          key_size = key.length
-  #          max_key_size = key_size if key_size > max_key_size
-
-  #          start_pos, end_pos = values
-  #          if Array === start_pos
-  #            start_pos.zip(end_pos).each do |s,e|
-  #              index_data << [key, [s.to_i, e.to_i]]
-  #            end
-  #          else
-  #            index_data << [key, [start_pos.to_i, end_pos.to_i]]
-  #          end
-  #        end
-  #      end
-  #    end
-
-  #    index = FixWidthTable.get(:memory, max_key_size, true)
-  #    index.add_range index_data
-  #    index.read
-  #    index
-  #  end
-  #end
-
-  #def self.range_index(file, start_field = nil, end_field = nil, options = {})
-  #  start_field ||= "Start"
-  #  end_field ||= "End"
-
-  #  data_options = Misc.pull_keys options, :data
-  #  filename = case
-  #             when (String === file or Path === file)
-  #               file
-  #             when file.respond_to?(:filename)
-  #               file.filename
-  #             else
-  #               file.object_id.to_s
-  #             end
-  #  persist_options = Misc.pull_keys options, :persist
-  #  persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
-
-  #  filters = Misc.process_options options, :filters
-
-  #  if filters
-  #    filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
-  #  end
-
-  #  Persist.persist(filename, :fwt, persist_options) do
-  #    tsv = TSV.open(file, data_options)
-  #    if filters
-  #      tsv.filter
-  #      filters.each do |match, value|
-  #        tsv.add_filter match, value
-  #      end
-  #    end
- 
-  #    tsv.range_index(start_field, end_field, options)
-  #  end
-  #end
 end

data/lib/scout/tsv/open.rb

@@ -34,7 +34,7 @@ module Open
   end
 
   def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, keep_open: false, **options, &block)
-    cpus = nil if cpus == 1
+    cpus = nil if cpus.to_i == 1
 
     if into == :stream
       sout, sin = Open.pipe
@@ -68,7 +68,15 @@ module Open
             bar.remove($!) if bar
           end
         end
+
         Thread.pass until into_thread["name"]
+
+        case into
+        when IO
+          ConcurrentStream.setup into, :threads => into_thread
+        when TSV::Dumper
+          ConcurrentStream.setup into.stream, :threads => into_thread
+        end
         return into
       end
     end
@@ -82,27 +90,29 @@ module Open
         callback.call res
       end
       
-      self.traverse(obj, **options) do |*args|
-        queue.write args
-      end
-
       begin
+        self.traverse(obj, **options) do |*args|
+          queue.write args
+        end
+
         queue.close
 
-        queue.join
+        queue.join(false)
 
         bar.remove if bar
         return into
       rescue Exception
         bar.remove($!) if bar
+        queue.abort
         raise $!
+      ensure
+        queue.clean
       end
     end
 
     begin
       res = case obj
             when TSV
-              #obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v,f|
               obj.traverse  unnamed: unnamed, **options do |k,v,f|
                 res = block.call(k, v, f)
                 callback.call res if callback
@@ -122,11 +132,21 @@ module Open
               raise obj.exception if obj.error?
               self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
             when IO
-              parser = TSV::Parser.new obj
-              parser.traverse **options do |k,v,f|
-                res = block.call k,v,f
-                callback.call res if callback
-                nil
+              if options[:type] == :array || options[:type] == :line
+                Log.low "Traverse stream by lines #{Log.fingerprint obj}"
+                while line = obj.gets
+                  line.strip!
+                  res = block.call(line)
+                  callback.call res if callback
+                end
+              else
+                Log.low "Traverse stream with parser #{Log.fingerprint obj}"
+                parser = TSV::Parser.new obj
+                parser.traverse **options do |k,v,f|
+                  res = block.call k,v,f
+                  callback.call res if callback
+                  nil
+                end
               end
             when TSV::Parser
               obj.traverse **options do |k,v,f|
@@ -142,9 +162,10 @@ module Open
               end
             end
       bar.remove if bar
-    rescue
+    rescue Exception => exception
+      exception = obj.stream_exception if (ConcurrentStream === obj) && obj.stream_exception
       bar.error if bar
-      raise $!
+      raise exception
     end
 
     into || res

data/lib/scout/tsv/parser.rb

@@ -190,6 +190,11 @@ module TSV
   end
 
   def self.parse_header(stream, fix: true, header_hash: '#', sep: "\t")
+    if (Path === stream) || ((String === stream) && Path.is_filename?(stream))
+      Open.open(stream) do |f|
+        return parse_header(f, fix: fix, header_hash: header_hash, sep: sep)
+      end
+    end
     raise "Closed stream" if IO === stream && stream.closed?
 
     opts = {}
@@ -240,7 +245,7 @@ module TSV
     opts[:type] = opts[:type].to_sym if opts[:type]
     opts[:cast] = opts[:cast].to_sym if opts[:cast]
 
-    [opts, key_field, fields, first_line, preamble]
+    NamedArray.setup([opts, key_field, fields, first_line, preamble], %w(options key_field fields first_line preamble))
   end
 
   KEY_PARAMETERS = begin
@@ -263,7 +268,7 @@ module TSV
       @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
       @options[:sep] = sep if @options[:sep].nil?
       @options.merge!(:key_field => @key_field, :fields => @fields)
-      @type = type
+      @type = @options[:type] || type
     end
 
     def all_fields
@@ -304,6 +309,8 @@ module TSV
         field_names = @fields
       end
 
+      kwargs[:positions] = nil if @type == :flat
+
       if key_field
         if @fields
           all_field_names ||= [@key_field] + @fields

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -42,6 +42,8 @@ module ScoutCabinet
 
     database.open(path, tokyocabinet_class::OREADER)
 
+    database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
+
     Persist::CONNECTIONS[path] = database
 
     database

data/lib/scout/tsv/stream.rb

@@ -0,0 +1,204 @@
+module TSV
+  def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, field_prefix: nil)
+
+    streams = streams.collect do |stream|
+      case stream
+      when(defined? Step and Step)
+        stream.stream
+      when Path
+        stream.open
+      when TSV::Dumper
+        stream.stream
+      else
+        stream
+      end
+    end.compact
+
+    num_streams = streams.length
+
+    streams = streams.collect do |stream|
+      Open.sort_stream(stream, memory: sort_memory)
+    end if sort
+
+    begin
+
+      lines         =[]
+      fields        =[]
+      sizes         =[]
+      key_fields    =[]
+      input_options =[]
+      empty         =[]
+      preambles     =[]
+      parser_types  =[]
+
+      type ||= :double
+
+      streams = streams.collect do |stream|
+
+        parser = TSV::Parser.new stream, type: type
+
+        sfields = parser.fields
+
+        if field_prefix
+          index = streams.index stream
+          prefix = field_prefix[index]
+
+          sfields = sfields.collect{|f|[prefix, f]* ":"}
+        end
+
+        first_line = parser.first_line
+        first_line = nil if first_line == ""
+
+        lines         << first_line
+        key_fields    << parser.key_field
+        fields        << sfields
+        sizes         << sfields.length if sfields
+        input_options << parser.options
+        preambles     << parser.preamble      if preamble and not parser.preamble.empty?
+        parser_types  << parser.type
+
+        empty         << stream               if parser.first_line.nil? || parser.first_line.empty?
+
+        stream
+      end
+
+
+      all_fields = fields.dup
+
+      key_field = key_fields.compact.first
+
+      if same_fields
+        fields = fields.first
+      else
+        fields = fields.compact.flatten
+      end
+
+      options = input_options.first 
+      type ||= options[:type]
+      type ||= :list if type == :single
+      type ||= :double if type == :flat
+
+      preamble_txt = case preamble
+                     when TrueClass
+                       preambles * "\n"
+                     when String
+                       if preamble[0]== '+'
+                         preambles * "\n" + "\n" + preamble[1..-1]
+                       else
+                         preamble
+                       end
+                     else
+                       nil
+                     end
+
+      empty_pos = empty.collect{|stream| streams.index stream}
+
+      keys =[]
+      parts =[]
+      lines.each_with_index do |line,i|
+        if line.nil? || line.empty?
+          keys[i]= nil
+          parts[i]= nil
+        else
+          vs = line.chomp.split(sep, -1)
+          key, *p = vs
+          keys[i]= key
+          parts[i]= p
+        end
+        sizes[i]||= parts[i].length-1 unless parts[i].nil?
+      end
+      done_streams =[]
+
+      dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type
+      dumper.init 
+
+      t = Thread.new do
+        Thread.report_on_exception = false
+        Thread.current["name"] = "Paste streams"
+
+        last_min = nil
+        while lines.reject{|line| line.nil?}.any?
+          min = keys.compact.sort.first
+          break if min.nil?
+          new_values =[]
+
+          skip = all_match && keys.uniq !=[min]
+
+          keys.each_with_index do |key,i|
+            case key
+            when min
+              new_values << parts[i]
+
+              begin
+                line = lines[i]= begin
+                                   streams[i].gets
+                               rescue
+                                 Log.exception $!
+                                 nil
+                               end
+              if line.nil?
+                keys[i]= nil
+                parts[i]= nil
+              else
+                k, *p = line.chomp.split(sep, -1)
+                raise TryAgain if k == keys[i]
+                keys[i]= k
+                parts[i]= p.collect{|e| e.nil? ? "" : e}
+              end
+            rescue TryAgain
+              keys[i]= nil
+              parts[i]= nil
+              Log.debug "Skipping repeated key in stream #{i}: #{keys[i]}"
+              retry
+            end
+          else
+            p = [nil] * sizes[i]
+            new_values << p
+          end
+        end
+
+        next if skip
+
+        if same_fields
+          new_values_same = []
+          new_values.each do |list|
+            list.each_with_index do |l,i|
+              new_values_same[i] ||= []
+              new_values_same[i] << l
+            end
+          end
+          new_values = new_values_same
+        else
+          new_values = new_values.inject([]){|acc,l| acc.concat l }
+        end
+
+        dumper.add min, new_values
+      end
+
+      dumper.close
+
+      streams.each do |stream|
+        stream.close if stream.respond_to?(:close) && ! stream.closed?
+        stream.join if stream.respond_to? :join
+      end
+      end
+    rescue Aborted
+      Log.error "Aborted pasting streams #{streams.inspect}: #{$!.message}"
+      streams.each do |stream|
+        stream.abort if stream.respond_to? :abort
+      end
+      raise $!
+    rescue Exception
+      Log.error "Exception pasting streams #{streams.inspect}: #{$!.message}"
+      streams.each do |stream|
+        stream.abort if stream.respond_to? :abort
+      end
+      raise $!
+    end
+
+    Thread.pass until t["name"]
+
+    ConcurrentStream.setup(dumper.stream, threads: [t])
+  end
+
+end

data/lib/scout/tsv/transformer.rb

@@ -137,5 +137,16 @@ module TSV
     end
     res
   end
+
+  def to_flat
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :flat
+    transformer.traverse do |k,v|
+      v = Array === v ? v.flatten : [v]
+      [k, v]
+    end
+    res
+  end
 end
 

data/lib/scout/tsv.rb

@@ -10,10 +10,11 @@ require_relative 'tsv/traverse'
 require_relative 'tsv/open'
 require_relative 'tsv/attach'
 require_relative 'tsv/change_id'
+require_relative 'tsv/stream'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
+  extension_attr :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers
 
   def self.str2options(str)
     field_options,_sep, rest =  str.partition("#")
@@ -40,7 +41,13 @@ module TSV
       data = filename ? ScoutCabinet.open(filename, true, type) : nil
       options[:data] = data if data
       options[:filename] = file
-      Log.debug "TSV open #{Log.fingerprint file}"
+
+      if data
+        Log.debug "TSV open #{Log.fingerprint file} into #{Log.fingerprint data}"
+      else
+        Log.debug "TSV open #{Log.fingerprint file}"
+      end
+
       Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
         TSV.parse(f, **options)
       end

data/lib/scout/work_queue/worker.rb

@@ -27,10 +27,10 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          Process.exit! -1
+          exit -1
         ensure
         end
-        Process.exit! 0
+        exit 0
       end
     end