scout-gear 7.3.0 → 8.0.0

data/lib/scout/tsv/open.rb

@@ -1,61 +1,56 @@
 require_relative '../open'
+require_relative '../work_queue'
+
+module MultipleResult
+  def self.setup(obj)
+    obj.extend MultipleResult
+    obj
+  end
+end
+
 module Open
   def self.traverse_add(into, res)
-    case into
-    when TSV::Dumper
-      into.add *res
-    when TSV, Hash
-      key, value = res
-      into[key] = value
+    if Array === res && MultipleResult === res
+      res.each do |_res|
+        traverse_add into, _res
+      end
+    else
+      case into
+      when defined?(TSV::Dumper) && TSV::Dumper
+        into.add *res
+      when TSV, Hash
+        key, value = res
+        if into.type == :double
+          into.zip_new key, value, insitu: false
+        else
+          into[key] = value
+        end
+      when Array, Set
+        into << res
+      when IO, StringIO
+        into.puts res
+      end
     end
   end
 
-  #def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
-  #  case obj
-  #  when TSV
-  #    obj.traverse options[:key_field], options[:fields], **options do |k,v|
-  #      res = yield k, v
-  #    end
-  #  when String
-  #    f = Open.open(obj)
-  #    self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
-  #  when Step
-  #    self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
-  #  when IO
-  #    if into && (IO === into || into.respond_to?(:stream) )
-  #      into_thread = Thread.new do 
-  #        Thread.current.report_on_exception = false
-  #        Thread.current["name"] = "Traverse into"
-  #        TSV.parse obj, **options do |k,v|
-  #          begin
-  #            res = block.call k, v
-  #            traverse_add into, res
-  #          rescue
-  #            into.abort $!
-  #          end
-  #          nil
-  #        end
-  #        into.close if into.respond_to?(:close)
-  #      end
-  #      Thread.pass until into_thread
-  #      into
-  #    else
-  #      TSV.parse obj, **options do |k,v|
-  #        block.call k, v
-  #        nil
-  #      end
-  #    end
-  #  end
-  #end
-
-  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, **options, &block)
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, keep_open: false, **options, &block)
+    cpus = nil if cpus == 1
+
+    if into == :stream
+      sout, sin = Open.pipe
+      ConcurrentStream.setup(sout, :pair => sin)
+      ConcurrentStream.setup(sin, :pair => sout)
+      self.traverse(obj, into: sin, cpus: cpus, bar: bar, callback: callback, unnamed: unnamed, **options, &block)
+      return sout
+    end
 
     if into || bar
       orig_callback = callback if callback
-      bar = Log::ProgressBar.get_obj_bar(bar, obj)
+      bar = Log::ProgressBar.get_obj_bar(obj, bar) if bar
+      bar.init if bar
       callback = proc do |res|
         bar.tick if bar
-        traverse_add into, res if into
+        traverse_add into, res if into && ! res.nil?
         orig_callback.call res if orig_callback
       end
 
@@ -65,59 +60,94 @@ module Open
           Thread.current["name"] = "Traverse into"
           error = false
           begin
-            self.traverse(obj, callback: callback, **options, &block)
-            into.close if into.respond_to?(:close)
+            self.traverse(obj, callback: callback, cpus: cpus, unnamed: unnamed, **options, &block)
+            into.close if ! keep_open && into.respond_to?(:close)
             bar.remove if bar
           rescue Exception
             into.abort($!) if into.respond_to?(:abort)
             bar.remove($!) if bar
           end
         end
-        Thread.pass until into_thread
+        Thread.pass until into_thread["name"]
         return into
       end
     end
 
-    begin
-      case obj
-      when TSV
-        obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v|
-          res = block.call(k, v)
-          callback.call res if callback
-          nil
-        end
-      when Array
-        obj.each do |line|
-          res = block.call(line)
-          callback.call res if callback
-          nil
-        end
-      when String
-        f = Open.open(obj)
-        self.traverse(f, cpus: cpus, callback: callback, **options, &block)
-      when Step
-        raise obj.exception if obj.error?
-        self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
-      when IO
-        TSV.parse obj, **options do |k,v|
-          res = block.call k, v
-          callback.call res if callback
-          nil
-        end
-      else
-        TSV.parse obj, **options do |k,v|
-          res = block.call k, v
-          callback.call res if callback
-          nil
-        end
+    if cpus
+      queue = WorkQueue.new cpus do |args|
+        block.call *args
+      end
+
+      queue.process do |res|
+        callback.call res
+      end
+      
+      self.traverse(obj, **options) do |*args|
+        queue.write args
+      end
+
+      begin
+        queue.close
+
+        queue.join
+
+        bar.remove if bar
+        return into
+      rescue Exception
+        bar.remove($!) if bar
+        raise $!
       end
+    end
+
+    begin
+      res = case obj
+            when TSV
+              #obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v,f|
+              obj.traverse  unnamed: unnamed, **options do |k,v,f|
+                res = block.call(k, v, f)
+                callback.call res if callback
+                nil
+              end
+            when Array
+              obj.each do |line|
+                res = block.call(line)
+                callback.call res if callback
+                nil
+              end
+            when String
+              obj = obj.produce_and_find if Path === obj
+              f = Open.open(obj)
+              self.traverse(f, cpus: cpus, callback: callback, **options, &block)
+            when Step
+              raise obj.exception if obj.error?
+              self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
+            when IO
+              parser = TSV::Parser.new obj
+              parser.traverse **options do |k,v,f|
+                res = block.call k,v,f
+                callback.call res if callback
+                nil
+              end
+            when TSV::Parser
+              obj.traverse **options do |k,v,f|
+                res = block.call k, v, f
+                callback.call res if callback
+                nil
+              end
+            else
+              TSV.parse obj, **options do |k,v|
+                res = block.call k, v
+                callback.call res if callback
+                nil
+              end
+            end
       bar.remove if bar
     rescue
-      bar.abort($!) if bar
+      bar.error if bar
       raise $!
     end
 
-    into
+    into || res
   end
 end
 
@@ -125,4 +155,28 @@ module TSV
   def self.traverse(*args, **kwargs, &block)
     Open.traverse(*args, **kwargs, &block)
   end
+
+  def self.process_stream(stream, header_hash: "#", &block)
+    sout = Open.open_pipe do |sin|
+      while line = stream.gets 
+        break unless line.start_with?(header_hash)
+        sin.puts line
+      end
+      yield sin, line
+    end
+  end
+
+  def self.collapse_stream(stream, *args, **kwargs, &block)
+    stream = stream.stream if stream.respond_to?(:stream)
+    self.process_stream(stream) do |sin, line|
+      collapsed = Open.collapse_stream(stream, line: line)
+      Open.consume_stream(collapsed, false, sin)
+    end
+  end
+
+  def collapse_stream(*args, **kwargs, &block)
+    TSV.collapse_stream(self.dumper_stream, *args, **kwargs, &block)
+  end
+
+
 end

data/lib/scout/tsv/parser.rb

@@ -12,17 +12,24 @@ module TSV
     end
   end
 
-  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil)
+  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil, select: nil, field_names: nil)
     items = line.split(sep, -1)
 
+    return nil if select && ! TSV.select(items[0], items[1..-1], select, fields: field_names, type: type, sep: sep2)
+
     if positions.nil? && key == 0
       key = items.shift
-    elsif positions.nil? 
-      key = items.delete_at(key)
+    elsif positions.nil?
+      if type == :flat
+        key = items[1..-1].collect{|e| e.split(sep2, -1) }.flatten
+        items = items.slice(0,1)
+      else
+        key = items.delete_at(key)
+      end
       key = key.split(sep2) if type == :double
     else 
       key, items = items[key], items.values_at(*positions)
-      key = key.split(sep2) if type == :double
+      key = key.split(sep2) if type == :double || type == :flat
     end
 
     items = case type
@@ -31,9 +38,9 @@ module TSV
             when :single
               items.first
             when :flat
-              [items]
+              items.collect{|i| i.split(sep2, -1) }.flatten
             when :double
-              items.collect{|i| i.split(sep2, -1) }
+              items.collect{|i| i.nil? ? [] : i.split(sep2, -1) }
             end
 
 
@@ -44,21 +51,34 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, **kargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, **kargs, &block)
     begin
-      bar = Log::ProgressBar.new_bar(bar) if bar
+      bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
+      bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
+      bar.init if bar
 
       source_type = type if source_type.nil?
 
       data = {} if data.nil?
-      merge = false if type != :double
+      merge = false if type != :double && type != :flat
       line = first_line || stream.gets
       while line
         begin
-          line.strip!
-          line = Misc.fixutf8(line) if fix
+          line.chomp!
+          if Proc === fix
+            line = fix.call line
+          elsif fix
+            line = Misc.fixutf8(line)
+          end
           bar.tick if bar
-          key, items = parse_line(line, type: source_type, **kargs)
+          if type == :array || type == :line
+            block.call line
+            next
+          end
+
+          key, items = parse_line(line, type: source_type, field_names: field_names, **kargs)
+
+          next if key.nil?
 
           if Array === key
             keys = key
@@ -116,44 +136,63 @@ module TSV
               end
 
             if block_given?
-              res = block.call(key, these_items)
+              res = block.call(key, these_items, field_names)
               data[key] = res unless res.nil? || FalseClass === data
               next
             end
 
             if ! merge || ! data.include?(key)
               data[key] = these_items
-            else
+            elsif type == :double
               current = data[key]
               if merge == :concat
                 these_items.each_with_index do |new,i|
-                  next if new.empty?
+                  new = [nil] if new.empty?
                   current[i].concat(new)
                 end
               else
                 merged = []
                 these_items.each_with_index do |new,i|
-                  next if new.empty?
+                  new = [nil] if new.empty?
                   merged[i] = current[i] + new
                 end
                 data[key] = merged
               end
+            elsif type == :flat
+              current = data[key]
+              if merge == :concat
+                current[i].concat these_items
+              else
+                data[key] = current + these_items
+              end
             end
           end
+        rescue Exception
+          stream.abort($!) if stream.respond_to?(:abort)
+          raise $!
         ensure
-          line = stream.gets
+          if stream.closed?
+            line = nil
+          else
+            line = stream.gets 
+          end
         end
       end
       data
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
+      if stream.stream_exception
+        bar.remove(stream.stream_exception)
+      else
+        bar.remove
+      end if bar
+      stream.join if stream.respond_to?(:join)
     end
   end
 
   def self.parse_header(stream, fix: true, header_hash: '#', sep: "\t")
     raise "Closed stream" if IO === stream && stream.closed?
 
-    options = {}
+    opts = {}
     preamble = []
 
     # Get line
@@ -165,13 +204,19 @@ module TSV
 
     # Process options line
     if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
-      options = IndiferentHash.string2hash m.captures.first.chomp
+      opts = IndiferentHash.string2hash m.captures.first.chomp
       line = stream.gets
-      line = Misc.fixutf8 line.chomp if line && fix
+      if line && fix
+        if Proc === fix
+          line = fix.call line
+        else
+          line = Misc.fixutf8 line.chomp if line && fix
+        end
+      end
     end
 
     # Determine separator
-    sep = options[:sep] if options[:sep]
+    sep = opts[:sep] if opts[:sep]
 
     # Process fields line
     preamble << line if line
@@ -192,7 +237,10 @@ module TSV
 
     first_line = line
 
-    [options, key_field, fields, first_line, preamble]
+    opts[:type] = opts[:type].to_sym if opts[:type]
+    opts[:cast] = opts[:cast].to_sym if opts[:cast]
+
+    [opts, key_field, fields, first_line, preamble]
   end
 
   KEY_PARAMETERS = begin
@@ -204,43 +252,79 @@ module TSV
                    end
 
   class Parser
-    attr_accessor :stream, :options, :key_field, :fields, :first_line, :preamble
-    def initialize(file, fix: true, header_hash: "#", sep: "\t")
+    attr_accessor :stream, :options, :key_field, :fields, :type, :first_line, :preamble
+    def initialize(file, fix: true, header_hash: "#", sep: "\t", type: :double)
       if IO === file
         @stream = file
       else
         @stream = Open.open(file)
       end
+      @fix = fix
       @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
       @options[:sep] = sep if @options[:sep].nil?
+      @options.merge!(:key_field => @key_field, :fields => @fields)
+      @type = type
     end
 
     def all_fields
+      return nil if @fields.nil?
       [@key_field] + @fields
     end
 
+    def key_field=(key_field)
+      @options[:key_field] = @key_field = key_field
+    end
+    
+    def fields=(fields)
+      @options[:fields] = @fields = fields
+    end
+
+    def identify_field(name)
+      TSV.identify_field(@key_field, @fields, name)
+    end
+
     def traverse(key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
+      kwargs[:type] ||=  self.options[:type] ||= @type
+      kwargs[:type] = kwargs[:type].to_sym if kwargs[:type]
+
       if fields
-        all_field_names ||= [@key_field] + @fields
-        positions = NamedArray.identify_name(all_field_names, fields)
-        kwargs[:positions] = positions
-        field_names = all_field_names.values_at *positions
+        if @fields
+          all_field_names ||= [@key_field] + @fields
+          fields = all_field_names if fields == :all
+          positions = NamedArray.identify_name(all_field_names, fields)
+          kwargs[:positions] = positions
+          field_names = all_field_names.values_at *positions
+        elsif fields.reject{|f| Numeric === f}.empty?
+          positions = fields
+          kwargs[:positions] = positions
+        else
+          raise "Non-numeric fields specified, but no field names available"
+        end
       else
         field_names = @fields
       end
 
       if key_field
-        all_field_names ||= [@key_field] + @fields
-        key = NamedArray.identify_name(all_field_names, key_field)
-        kwargs[:key] = key == :key ? 0 : key
-        key_field_name = key === :key ? @key_field : all_field_names[key]
-        if fields.nil?
-          field_names = all_field_names - [@key_field]
+        if @fields
+          all_field_names ||= [@key_field] + @fields
+          key = NamedArray.identify_name(all_field_names, key_field)
+          kwargs[:key] = key == :key ? 0 : key
+          key_field_name = key === :key ? @key_field : all_field_names[key]
+          if fields.nil?
+            field_names = all_field_names - [key_field_name]
+          end
+        else
+          kwargs[:key] = key_field == :key ? 0 : key_field
+          key = key_field
         end
       else
         key_field_name = @key_field
       end
 
+      if field_names && (kwargs[:type] == :single || kwargs[:type] == :flat)
+        field_names = field_names.slice(0,1)
+      end
+
       @options.each do |option,value|
         option = option.to_sym
         next unless KEY_PARAMETERS.include? option
@@ -250,21 +334,33 @@ module TSV
       kwargs[:source_type] = @options[:type]
       kwargs[:data] = false if kwargs[:data].nil?
 
-      data = TSV.parse_stream(@stream, first_line: @first_line, **kwargs, &block)
+      data = TSV.parse_stream(@stream, first_line: @first_line, fix: @fix, field_names: @fields, **kwargs, &block)
 
       if data
         TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
       else
-        self
+        [self.key_field, self.fields]
       end
     end
 
+    def fingerprint
+      "Parser:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+    end
+
+    def digest_str
+      fingerprint
+    end
+
+    def inspect
+      fingerprint
+    end
   end
 
   def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
     parser = TSV::Parser.new stream, fix: fix, header_hash: header_hash, sep: sep
 
-    cast = parser.options[:cast] || kwargs[:cast]
+    cast = kwargs[:cast]
+    cast = parser.options[:cast] if cast.nil?
     type = kwargs[:type] ||=  parser.options[:type] ||= :double
     if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
       TSV.setup(data, type: type)
@@ -297,48 +393,7 @@ module TSV
     data.filename = filename
     data.namespace = namespace
     data.unnamed = unnamed
+    data.save_extension_attr_hash if data.respond_to?(:save_extension_attr_hash)
     data
   end
-
-  #def self.parse_alt(stream, key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
-  #  options, key_field_name, field_names, first_line, preamble = parse_header(stream)
-
-  #  if fields
-  #    all_field_names ||= [key_field_name] + field_names
-  #    positions = NamedArray.identify_name(all_field_names, fields)
-  #    kwargs[:positions] = positions
-  #    field_names = all_field_names.values_at *positions
-  #  end
-
-  #  if key_field
-  #    all_field_names ||= [key_field_name] + field_names
-  #    key = NamedArray.identify_name(all_field_names, key_field)
-  #    kwargs[:key] = key
-  #    key_field_name = all_field_names[key]
-  #    if fields.nil?
-  #      field_names = all_field_names - [key_field_name]
-  #    end
-  #  end
-
-  #  options.each do |option,value|
-  #    option = option.to_sym
-  #    next unless KEY_PARAMETERS.include? option
-  #    kwargs[option] = value unless kwargs.include?(option)
-  #  end
-
-  #  kwargs[:source_type] = options[:type]
-
-  #  type = kwargs[:type] ||= :double
-  #  if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
-  #    TSV.setup(data, type: type, key_field: key_field_name, fields: field_names)
-  #    data.extend TSVAdapter
-  #  end
-
-  #  data = parse_stream(stream, first_line: first_line, **kwargs, &block)
-
-  #  TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => type, filename: filename, namespace: namespace)
-
-  #  data
-  #end
-
 end

data/lib/scout/tsv/path.rb

@@ -1,7 +1,6 @@
 module Path
   def tsv(*args, **kwargs, &block)
-    found = self.find
-    found = self.set_extension('tsv').find unless found.exists?
+    found = produce_and_find('tsv')
     TSV.open(found, *args, **kwargs, &block)
   end