scout-gear 7.3.0 → 8.1.0

data/lib/scout/open/stream.rb

@@ -3,7 +3,7 @@ module Open
 
   class << self
     attr_accessor :sensible_write_lock_dir
-    
+
     def sensible_write_lock_dir
       @sensible_write_lock_dir ||= Path.setup("tmp/sensible_write_locks").find
     end
@@ -18,7 +18,7 @@ module Open
 
   def self.consume_stream(io, in_thread = false, into = nil, into_close = true, &block)
     return if Path === io
-    return unless io.respond_to? :read 
+    return unless io.respond_to? :read
 
     if io.respond_to? :closed? and io.closed?
       io.join if io.respond_to? :join
@@ -46,16 +46,17 @@ module Open
       begin
         into = into.find if Path === into
 
-        if String === into 
+        if String === into
           dir = File.dirname(into)
           Open.mkdir dir unless File.exist?(dir)
-          into_path, into = into, File.open(into, 'w') 
+          into_path, into = into, File.open(into, 'w')
         end
-        
+
         into_close = false unless into.respond_to? :close
 
         while c = io.read(BLOCK_SIZE)
           into << c if into
+          last_c = c if c
           break if io.closed?
         end
 
@@ -65,20 +66,23 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        c
+        Log.debug "Consume stream done #{Log.fingerprint io} -> #{Log.fingerprint(into_path ||into)}"
+        last_c
       rescue Aborted
+        Thread.current["exception"] = true
         Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
+        Thread.current["exception"] = true
         Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
         exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         into_path = into if into_path.nil? && String === into
         if into_path and File.exist?(into_path)
-          FileUtils.rm into_path 
+          FileUtils.rm into_path
         end
         raise exception
       end
@@ -89,7 +93,7 @@ module Open
     force = IndiferentHash.process_options options, :force
 
     if File.exist?(path) and not force
-      Open.consume_stream content 
+      Open.consume_stream content
       return
     end
 
@@ -104,12 +108,12 @@ module Open
 
       if File.exist? path and not force
         Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
-        Open.consume_stream content 
+        Open.consume_stream content
       else
         FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
+        Log.low "Sensible write stream #{Log.fingerprint content} -> #{Log.fingerprint path}" if IO === content
         begin
-
           case
           when block_given?
             File.open(tmp_path, 'wb', &block)
@@ -117,11 +121,10 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              #f.sync = true
               while block = content.read(BLOCK_SIZE)
                 f.write block
                 break if content.closed?
-              end 
+              end
             end
           else
             File.open(tmp_path, 'wb') do |f|  end
@@ -138,15 +141,15 @@ module Open
           Open.touch path if File.exist? path
           content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
 
-          Open.notify_write(path) 
+          Open.notify_write(path)
         rescue Aborted
-          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
-          content.abort if content.respond_to? :abort
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ path }"
+          content.abort(exception) if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
         rescue
@@ -172,7 +175,7 @@ module Open
 
       [sout, sin]
     end
-    Log.debug{"Creating pipe #{[Log.fingerprint(res.last), Log.fingerprint(res.first)] * " => "}"}
+    Log.low{"Creating pipe #{[Log.fingerprint(res.last), Log.fingerprint(res.first)] * " -> "}"}
     res
   end
 
@@ -187,7 +190,7 @@ module Open
       FileUtils.rm path if erase && File.exist?(path)
     end
   end
-  
+
   def self.release_pipes(*pipes)
     PIPE_MUTEX.synchronize do
       pipes.flatten.each do |pipe|
@@ -212,14 +215,13 @@ module Open
 
     if do_fork
 
-      #parent_pid = Process.pid
       pid = Process.fork {
         begin
           purge_pipes(sin)
           sout.close
 
           yield sin
-          sin.close if close and not sin.closed? 
+          sin.close if close and not sin.closed?
 
         rescue Exception
           Log.exception $!
@@ -235,34 +237,20 @@ module Open
       ConcurrentStream.setup sin, :pair => sout
       ConcurrentStream.setup sout, :pair => sin
 
-      thread = Thread.new do 
+      thread = Thread.new do
         begin
-          Thread.current.report_on_exception = false
-          Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-          
-          yield sin
+          ConcurrentStream.process_stream(sin, :message => "Open pipe") do
+            Thread.current.report_on_exception = false
+            Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
 
-          sin.close if close and not sin.closed? and not sin.aborted?
-        rescue Aborted
-          Log.low "Aborted open_pipe: #{$!.message}"
-          raise $!
-        rescue Exception
-          Log.low "Exception in open_pipe: #{$!.message}"
-          begin
-            sout.threads.delete(Thread.current)
-            sout.pair = []
-            sout.abort($!) if sout.respond_to?(:abort)
-            sin.threads.delete(Thread.current)
-            sin.pair = []
-            sin.abort($!) if sin.respond_to?(:abort)
-          ensure
-            raise $!
+            yield sin
           end
         end
       end
 
       sin.threads = [thread]
       sout.threads = [thread]
+
       Thread.pass until thread["name"]
     end
 
@@ -272,12 +260,14 @@ module Open
   def self.tee_stream_thread_multiple(stream, num = 2)
     in_pipes = []
     out_pipes = []
-    num.times do 
+    num.times do
       sout, sin = Open.pipe
       in_pipes << sin
       out_pipes << sout
     end
 
+    Log.low("Tee stream #{Log.fingerprint stream} -> #{Log.fingerprint out_pipes}")
+
     filename = stream.filename if stream.respond_to? :filename
 
     splitter_thread = Thread.new(Thread.current) do |parent|
@@ -289,7 +279,7 @@ module Open
         while block = stream.read(BLOCK_SIZE)
 
           in_pipes.each_with_index do |sin,i|
-            begin 
+            begin
               sin.write block
             rescue IOError
               Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
@@ -297,18 +287,24 @@ module Open
             rescue
               Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
               raise $!
-            end unless skip[i] 
+            end unless skip[i]
           end
           break if stream.closed?
         end
 
         stream.join if stream.respond_to? :join
-        stream.close unless stream.closed?
         in_pipes.first.close unless in_pipes.first.closed?
       rescue Aborted, Interrupt
-        stream.abort if stream.respond_to? :abort
-        out_pipes.each do |sout|
-          sout.abort if sout.respond_to? :abort
+        stream.abort if stream.respond_to?(:abort) && ! stream.aborted?
+        out_pipes.reverse.each do |sout|
+          sout.threads.delete(Thread.current)
+          begin
+            sout.abort($!) if sout.respond_to?(:abort) && ! sout.aborted?
+          rescue
+          end
+        end
+        in_pipes.each do |sin|
+          sin.close unless sin.closed?
         end
         Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
@@ -327,28 +323,44 @@ module Open
           end
           Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
-          Log.exception $!
         ensure
-          in_pipes.each do |sin|
-            sin.close unless sin.closed?
+          begin
+            in_pipes.each do |sin|
+              sin.close unless sin.closed?
+            end
+          ensure
+            raise $!
           end
-          raise $!
         end
       end
     end
 
-    out_pipes.each do |sout|
-      ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
-    end
     Thread.pass until splitter_thread["name"]
 
     main_pipe = out_pipes.first
-    main_pipe.autojoin = true
 
-    main_pipe.callback = Proc.new do 
-      stream.join if stream.respond_to? :join
-      in_pipes[1..-1].each do |sin|
-        sin.close unless sin.closed?
+    ConcurrentStream.setup(main_pipe, :threads => [splitter_thread], :filename => filename, :autojoin => true)
+
+    out_pipes[1..-1].each do |sout|
+      ConcurrentStream.setup sout, :filename => filename, :threads => [splitter_thread]
+    end
+
+    main_pipe.callback = proc do
+      begin
+        stream.join if stream.respond_to?(:join) && ! stream.joined?
+        in_pipes[1..-1].each do |sin|
+          sin.close unless sin.closed?
+        end
+      rescue
+        main_pipe.abort_callback.call($!)
+        raise $!
+      end
+    end
+
+    main_pipe.abort_callback = proc do |exception|
+      stream.abort(exception)
+      out_pipes[1..-1].each do |sout|
+        sout.abort(exception)
       end
     end
 
@@ -367,7 +379,7 @@ module Open
     str = nil
     Thread.pass while IO.select([stream],nil,nil,1).nil?
     while not str = stream.read(size)
-      IO.select([stream],nil,nil,1) 
+      IO.select([stream],nil,nil,1)
       Thread.pass
       raise ClosedStream if stream.eof?
     end
@@ -392,38 +404,89 @@ module Open
     str
   end
 
-  def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
-    Open.open_pipe do |sin|
-      line = stream.gets
-      while line =~ /^#{header_hash}/ do
-        sin.puts line
+  def self.sort_stream(stream, header_hash: "#", cmd_args: "-u", memory: false)
+    sout = Open.open_pipe do |sin|
+      ConcurrentStream.process_stream(stream) do
         line = stream.gets
-      end
+        while line && line.start_with?(header_hash) do
+          sin.puts line
+          line = stream.gets
+        end
 
-      line_stream = Open.open_pipe do |line_stream_in|
-        line_stream_in.puts line
-        begin
+        line_stream = Open.open_pipe do |line_stream_in|
+          line_stream_in.puts line if line
           Open.consume_stream(stream, false, line_stream_in)
-        rescue
-          raise $!
         end
-      end
-
-      sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+        Log.low "Sub-sort stream #{Log.fingerprint stream} -> #{Log.fingerprint line_stream}"
 
-      begin
-        Open.consume_stream(sorted, false, sin)
-      rescue
-        Log.exception $!
-        begin
-          sorted.raise($!) if sorted.respond_to? :raise
-          stream.raise($!) if stream.respond_to? :raise
-        ensure
-          raise $!
+        if memory
+          line_stream.read.split("\n").sort.each do |line|
+            sin.puts line
+          end
+        else
+          io = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+          Open.consume_stream(io, false, sin)
         end
       end
     end
+    Log.low "Sort #{Log.fingerprint stream} -> #{Log.fingerprint sout}"
+    sout
   end
 
+  #def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
+  #  StringIO.new stream.read.split("\n").sort.uniq * "\n"
+  #end
+
+  def self.collapse_stream(s, line: nil, sep: "\t", header: nil, &block)
+    sep ||= "\t"
+    Open.open_pipe do |sin|
 
+      sin.puts header if header
+
+      line ||= s.gets
+
+      current_parts = []
+      while line
+        key, *parts = line.chomp.split(sep, -1)
+        case
+        when key.nil?
+        when current_parts.nil?
+          current_parts = parts
+          current_key = key
+        when current_key == key
+          parts.each_with_index do |part,i|
+            if current_parts[i].nil?
+              current_parts[i] = "|" << part
+            else
+              current_parts[i] = current_parts[i] << "|" << part
+            end
+          end
+
+          (parts.length..current_parts.length-1).to_a.each do |pos|
+            current_parts[pos] = current_parts[pos] << "|" << ""
+          end
+        when current_key.nil?
+          current_key = key
+          current_parts = parts
+        when current_key != key
+          if block_given?
+            res = block.call(current_parts)
+            sin.puts [current_key, res] * sep
+          else
+            sin.puts [current_key, current_parts].flatten * sep
+          end
+          current_key = key
+          current_parts = parts
+        end
+        line = s.gets
+      end
+
+      if block_given?
+        res = block.call(current_parts)
+        sin.puts [current_key, res] * sep
+      else
+        sin.puts [current_key, current_parts].flatten * sep
+      end unless current_key.nil?
+    end
+  end
 end

data/lib/scout/open/util.rb

@@ -68,6 +68,15 @@ module Open
     !! (file =~ /\.zip$/)
   end
 
+  def self.is_stream?(obj)
+    IO === obj || StringIO === obj
+  end
+
+  def self.has_stream?(obj)
+    obj.respond_to?(:stream)
+  end
+
+
   def self.notify_write(file)
     begin
       notification_file = file + '.notify'
@@ -94,6 +103,11 @@ module Open
     File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
+  def self.directory?(file)
+    file = file.find if Path === file
+    File.directory?(file)
+  end
+
   def self.exists?(file)
     file = file.find if Path === file
     File.exist?(file)
@@ -111,7 +125,7 @@ module Open
   end
 
   def self.rm(file)
-    FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
+    FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
   end
 
   def self.rm_rf(file)
@@ -155,7 +169,7 @@ module Open
     begin
       if File.symlink?(file) || File.stat(file).nlink > 1
         if File.exist?(file + '.info') && defined?(Step)
-          done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
+          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
           return done if done
         end
 
@@ -173,7 +187,7 @@ module Open
     target = target.find if Path === target
 
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm_rf target if File.exist?(target)
     FileUtils.cp_r source, target
   end
 
@@ -222,4 +236,9 @@ module Open
     end
     nil
   end
+
+  def self.list(file)
+    file = file.produce_and_find if Path === file
+    Open.read(file).split("\n")
+  end
 end

data/lib/scout/open.rb

@@ -21,6 +21,8 @@ module Open
   end
 
   def self.get_stream(file, mode = 'r')
+    return file if Open.is_stream?(file)
+    return file.stream if Open.has_stream?(file)
     file = file.find if Path === file
 
     return Open.ssh(file) if Open.ssh?(file)
@@ -56,11 +58,11 @@ module Open
   def self.open(file, options = {})
     if IO === file || StringIO === file
       if block_given?
-        res = yield file 
+        res = yield file
         file.close
         return res
       else
-        return file 
+        return file
       end
     end
 
@@ -163,7 +165,6 @@ module Open
       raise "Content unknown #{Log.fingerprint content}"
     end
 
-    notify_write(file) 
+    notify_write(file)
   end
-
 end

data/lib/scout/path/find.rb

@@ -10,8 +10,10 @@ module Path
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
           file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /scout\/resource\/produce.rb/ or
           file =~ /modules\/rbbt-util/
       end
+      return nil if file.nil?
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
 
@@ -32,13 +34,14 @@ module Path
 
   def self.follow(path, map, map_name = nil)
     file = map.sub('{PKGDIR}', path.pkgdir.respond_to?(:pkgdir) ? path.pkgdir.pkgdir || Path.default_pkgdir : path.pkgdir || Path.default_pkgdir).
+      sub('{HOME}', ENV["HOME"]).
       sub('{RESOURCE}', path.pkgdir.to_s).
       sub('{PWD}', FileUtils.pwd).
       sub('{TOPLEVEL}', path._toplevel).
       sub('{SUBPATH}', path._subpath).
       sub('{BASENAME}', File.basename(path)).
       sub('{PATH}', path).
-      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
+      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir || "NOLIBDIR").
       sub('{MAPNAME}', map_name.to_s).
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '').gsub(/\/+/,'/')
@@ -57,16 +60,17 @@ module Path
 
   def self.path_maps
     @@path_maps ||= IndiferentHash.setup({
-      :current => File.join("{PWD}", "{TOPLEVEL}", "{SUBPATH}"),
-      :user    => File.join(ENV['HOME'], ".{PKGDIR}", "{TOPLEVEL}", "{SUBPATH}"),
-      :global  => File.join('/', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :usr     => File.join('/usr/', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :local   => File.join('/usr/local', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :fast    => File.join('/fast', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :cache   => File.join('/cache', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :bulk    => File.join('/bulk', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
-      :lib     => File.join('{LIBDIR}', "{TOPLEVEL}", "{SUBPATH}"),
-      :base    => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}", "{SUBPATH}"),
+      :current => "{PWD}/{TOPLEVEL}/{SUBPATH}",
+      :user    => "{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}",
+      :global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+      :lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
+      :scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+      :tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
       :default => :user
     })
   end
@@ -125,6 +129,9 @@ module Path
   def follow(map_name = :default, annotate = true)
     IndiferentHash.setup(path_maps)
     map = path_maps[map_name] || Path.path_maps[map_name]
+    if map.nil? && String === map_name
+      map = File.join(map_name, '{TOPLEVEL}/{SUBPATH}')
+    end
     raise "Map not found #{Log.fingerprint map_name} not in #{Log.fingerprint path_maps.keys}" if map.nil?
     while Symbol === map
       map_name = map
@@ -188,4 +195,10 @@ module Path
       .select{|file| file.exist? }.uniq
   end
 
+  def find_with_extension(extension, *args)
+    found = self.find(*args)
+    return found if found.exists?
+    found_with_extension = self.set_extension(extension).find
+    found_with_extension.exists? ? found_with_extension : found
+  end
 end

data/lib/scout/path/util.rb

@@ -12,6 +12,23 @@ module Path
     return false
   end
 
+  def self.sanitize_filename(filename, length = 254)
+    if filename.length > length
+      if filename =~ /(\..{2,9})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
   def directory?
     return nil unless self.exist?
     File.directory?(self.find)
@@ -64,4 +81,27 @@ module Path
   def set_extension(extension)
     self.annotate(self + ".#{extension}")
   end
+
+  def unset_extension
+    self.annotate(self.split(".")[0..-2] * ".")
+  end
+
+
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
+    return true if not Open.exists?(file)
+    path = path.find if Path === path
+    file = file.find if Path === file
+    if by_link
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
+    return true if patht.nil? || filet.nil?
+    diff = patht - filet
+    return diff if diff < 0
+    return false
+  end
 end