scout-gear 7.3.0 → 8.0.0

data/lib/scout/open/stream.rb

@@ -56,6 +56,7 @@ module Open
 
         while c = io.read(BLOCK_SIZE)
           into << c if into
+          last_c = c if c
           break if io.closed?
         end
 
@@ -65,13 +66,15 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        c
+        last_c
       rescue Aborted
+        Thread.current["exception"] = true
         Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
+        Thread.current["exception"] = true
         Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
         exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
@@ -109,7 +112,6 @@ module Open
         FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
         begin
-
           case
           when block_given?
             File.open(tmp_path, 'wb', &block)
@@ -117,7 +119,6 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              #f.sync = true
               while block = content.read(BLOCK_SIZE)
                 f.write block
                 break if content.closed?
@@ -139,14 +140,15 @@ module Open
           content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
 
           Open.notify_write(path) 
+          Log.debug "Done sensible write: [#{Process.pid}] -- #{ path }"
         rescue Aborted
-          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
-          content.abort if content.respond_to? :abort
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ path }"
+          content.abort(exception) if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
         rescue
@@ -187,7 +189,7 @@ module Open
       FileUtils.rm path if erase && File.exist?(path)
     end
   end
-  
+
   def self.release_pipes(*pipes)
     PIPE_MUTEX.synchronize do
       pipes.flatten.each do |pipe|
@@ -239,7 +241,7 @@ module Open
         begin
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-          
+
           yield sin
 
           sin.close if close and not sin.closed? and not sin.aborted?
@@ -306,9 +308,16 @@ module Open
         stream.close unless stream.closed?
         in_pipes.first.close unless in_pipes.first.closed?
       rescue Aborted, Interrupt
-        stream.abort if stream.respond_to? :abort
-        out_pipes.each do |sout|
-          sout.abort if sout.respond_to? :abort
+        stream.abort if stream.respond_to?(:abort) && ! stream.aborted?
+        out_pipes.reverse.each do |sout|
+          sout.threads.delete(Thread.current)
+          begin
+            sout.abort($!) if sout.respond_to?(:abort) && ! sout.aborted?
+          rescue
+          end
+        end
+        in_pipes.each do |sin|
+          sin.close unless sin.closed?
         end
         Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
@@ -327,12 +336,14 @@ module Open
           end
           Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
-          Log.exception $!
         ensure
-          in_pipes.each do |sin|
-            sin.close unless sin.closed?
+          begin
+            in_pipes.each do |sin|
+              sin.close unless sin.closed?
+            end
+          ensure
+            raise $!
           end
-          raise $!
         end
       end
     end
@@ -425,5 +436,69 @@ module Open
     end
   end
 
+  def self.process_stream(s)
+    begin
+      yield s
+      s.close if s.respond_to?(:close) && ! s.closed?
+      s.join if s.respond_to?(:join)
+    rescue
+      s.abort($!) if s.respond_to? :abort
+      raise $!
+    end
+  end
+
+
+  def self.collapse_stream(s, line: nil, sep: "\t", header: nil, &block)
+    sep ||= "\t"
+    Open.open_pipe do |sin|
+      sin.puts header if header
+      process_stream(s) do |s|
+        line ||= s.gets
+
+        current_parts = []
+        while line 
+          key, *parts = line.chomp.split(sep, -1)
+          case
+          when key.nil?
+          when current_parts.nil?
+            current_parts = parts
+            current_key = key
+          when current_key == key
+            parts.each_with_index do |part,i|
+              if current_parts[i].nil?
+                current_parts[i] = "|" << part
+              else
+                current_parts[i] = current_parts[i] << "|" << part
+              end
+            end
+
+            (parts.length..current_parts.length-1).to_a.each do |pos|
+              current_parts[pos] = current_parts[pos] << "|" << ""
+            end
+          when current_key.nil?
+            current_key = key
+            current_parts = parts
+          when current_key != key
+            if block_given?
+              res = block.call(current_parts)
+              sin.puts [current_key, res] * sep
+            else
+              sin.puts [current_key, current_parts].flatten * sep
+            end 
+            current_key = key
+            current_parts = parts
+          end
+          line = s.gets
+        end
+
+        if block_given?
+          res = block.call(current_parts)
+          sin.puts [current_key, res] * sep
+        else
+          sin.puts [current_key, current_parts].flatten * sep
+        end unless current_key.nil?
+      end
+    end
+  end
 
 end

data/lib/scout/open/util.rb

@@ -94,6 +94,11 @@ module Open
     File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
+  def self.directory?(file)
+    file = file.find if Path === file
+    File.directory?(file)
+  end
+
   def self.exists?(file)
     file = file.find if Path === file
     File.exist?(file)
@@ -111,7 +116,7 @@ module Open
   end
 
   def self.rm(file)
-    FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
+    FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
   end
 
   def self.rm_rf(file)
@@ -155,7 +160,7 @@ module Open
     begin
       if File.symlink?(file) || File.stat(file).nlink > 1
         if File.exist?(file + '.info') && defined?(Step)
-          done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
+          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
           return done if done
         end
 
@@ -173,7 +178,7 @@ module Open
     target = target.find if Path === target
 
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm_rf target if File.exist?(target)
     FileUtils.cp_r source, target
   end
 
@@ -222,4 +227,9 @@ module Open
     end
     nil
   end
+
+  def self.list(file)
+    file = file.produce_and_find if Path === file
+    Open.read(file).split("\n")
+  end
 end

data/lib/scout/path/find.rb

@@ -10,8 +10,10 @@ module Path
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
           file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /scout\/resource\/produce.rb/ or
           file =~ /modules\/rbbt-util/
       end
+      return nil if file.nil?
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
 
@@ -38,7 +40,7 @@ module Path
       sub('{SUBPATH}', path._subpath).
       sub('{BASENAME}', File.basename(path)).
       sub('{PATH}', path).
-      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
+      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir || "NOLIBDIR").
       sub('{MAPNAME}', map_name.to_s).
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '').gsub(/\/+/,'/')
@@ -188,4 +190,10 @@ module Path
       .select{|file| file.exist? }.uniq
   end
 
+  def find_with_extension(extension, *args)
+    found = self.find(*args)
+    return found if found.exists?
+    found_with_extension = self.set_extension(extension).find
+    found_with_extension.exists? ? found_with_extension : found
+  end
 end

data/lib/scout/path/util.rb

@@ -12,6 +12,23 @@ module Path
     return false
   end
 
+  def self.sanitize_filename(filename, length = 254)
+    if filename.length > length
+      if filename =~ /(\..{2,9})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
   def directory?
     return nil unless self.exist?
     File.directory?(self.find)
@@ -64,4 +81,22 @@ module Path
   def set_extension(extension)
     self.annotate(self + ".#{extension}")
   end
+
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
+    return true if not Open.exists?(file)
+    path = path.find if Path === path
+    file = file.find if Path === file
+    if by_link
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
+    return true if patht.nil? || filet.nil?
+    diff = patht - filet
+    return diff if diff < 0
+    return false
+  end
 end

data/lib/scout/persist/serialize.rb

@@ -1,5 +1,6 @@
 require_relative '../open'
 require_relative 'open'
+require 'set'
 
 module Persist
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
@@ -19,7 +20,7 @@ module Persist
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
     case type
-    when nil, :string, :integer, :float, :boolean, :file, :path
+    when nil, :string, :integer, :float, :boolean, :file, :path, :select, :folder
       if IO === content || StringIO === content
         content.read
       else
@@ -46,8 +47,9 @@ module Persist
   def self.deserialize(serialized, type)
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
+
     case type
-    when nil, :string, :file, :stream
+    when nil, :string, :file, :stream, :select, :folder
       serialized
     when :path
       Path.setup(serialized)
@@ -99,9 +101,18 @@ module Persist
         return save_drivers[type].call(file, content)
       end
     end
-    serialized = serialize(content, type)
-    Open.sensible_write(file, serialized, :force => true)
-    return nil
+
+    if type == :binary
+      content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
+      Open.open(path, :mode => 'wb') do |f|
+        f.puts content
+      end
+      content
+   else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized, :force => true)
+      return nil
+    end
   end
 
   def self.load(file, type = :serializer)
@@ -118,6 +129,8 @@ module Persist
     end
 
     case type
+    when :binary
+      Open.read(file, :mode => 'rb')
     when :yaml
       Open.yaml(file)
     when :json

data/lib/scout/persist.rb

@@ -21,7 +21,9 @@ module Persist
   def self.persistence_path(name, options = {})
     options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
     other_options = IndiferentHash.pull_keys options, :other
-    TmpFile.tmp_for_file(name, options, other_options)
+    name = name.filename if name.respond_to?(:filename) && name.filename
+    persist_options = {}
+    TmpFile.tmp_for_file(name, options.merge(persist_options), other_options)
   end
 
   MEMORY_CACHE = {}
@@ -31,25 +33,31 @@ module Persist
     return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
-    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
-
-    update = options[:update] || persist_options[:update]
-    update = Open.mtime(update) if Path === update
-    update = Open.mtime(file) >= update ? false : true if Time === update
-
     if type == :memory
       repo = options[:memory] || options[:repo] || MEMORY_CACHE
       repo[file] ||= yield
       return repo[file]
     end
 
+    update = options[:update] || persist_options[:update]
+    update = Open.mtime(update) if Path === update
+    update = Open.mtime(file) >= update ? false : true if Time === update
+
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
     Open.lock lockfile do |lock|
       if Open.exist?(file) && ! update
         Persist.load(file, type)
       else
-        return yield(file) if block.arity == 1
-        res = yield
         begin
+          file = file.find if Path === file
+          return yield(file) if block.arity == 1
+          res = yield
+
+          if res.nil?
+            return Persist.load(file, type)
+          end
+
           Open.rm(file)
 
           if IO === res || StringIO === res
@@ -73,16 +81,24 @@ module Persist
             res = pres unless pres.nil?
           end
         rescue
+          Thread.handle_interrupt(Exception => :never) do
+            if Open.exist?(file)
+              Log.debug "Failed persistence #{file} - erasing"
+              Open.rm file
+            else
+              Log.debug "Failed persistence #{file}"
+            end
+          end
           raise $! unless options[:canfail]
-          Log.debug "Could not persist #{type} on #{file}"
         end
         res
       end
     end
   end
 
-  def self.memory(name, *args, &block)
-    self.persist(name, :memory, *args, &block)
+  def self.memory(name, options = {}, &block)
+    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":"
+    self.persist(name, :memory, options, &block)
   end
 
 end

data/lib/scout/resource/path.rb

@@ -1,4 +1,47 @@
 module Path
+  def produce(force = false)
+    return self if ! force && (Open.exist?(self) || @produced)
+    begin
+      if Resource === self.pkgdir
+        self.pkgdir.produce self
+      else
+        false
+      end
+    rescue ResourceNotFound
+      false
+    rescue
+      message = $!.message
+      message = "No exception message" if message.nil? || message.empty?
+      Log.warn "Error producing #{self}: #{message}"
+      false
+    ensure
+      @produced = true
+    end
+  end
+
+  def produce_with_extension(extension, *args)
+    begin
+      self.produce(*args)
+    rescue Exception
+      exception = $!
+      begin
+        self.set_extension(extension).produce(*args)
+      rescue Exception
+        raise exception
+      end
+    end
+  end
+
+  def produce_and_find(extension = nil, *args)
+    if extension
+      found = find_with_extension(extension, *args)
+      found.exists? ? found : produce_with_extension(extension, *args)
+    else
+      found = find
+      found.exists? ? found : produce(*args)
+    end
+  end
+
   def relocate
     return self if Open.exists?(self)
     Resource.relocate(self)
@@ -17,4 +60,14 @@ module Path
   def write(*args, &block)
     Open.write(self.find, *args, &block)
   end
+
+  def list
+    found = produce_and_find('list')
+    Open.list(found)
+  end
+
+  def exists?
+    return true if Open.exists?(self.find)
+    self.produce
+  end
 end

data/lib/scout/resource/produce.rb

@@ -149,11 +149,3 @@ module Resource
   end
 
 end
-
-module Path
-  def produce(force = false)
-    return self if ! force && Open.exist?(self)
-    self.pkgdir.produce self if Resource === self.pkgdir
-    return self
-  end
-end

data/lib/scout/resource/util.rb

@@ -1,7 +1,8 @@
 module Resource
   def identify(path)
     return path unless path.start_with?("/")
-    path_maps = path.path_maps || self.path_maps || Path.path_maps
+    path_maps = path.path_maps if Path === path
+    path_maps ||= self.path_maps || Path.path_maps
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
 

data/lib/scout/tmpfile.rb

@@ -93,19 +93,18 @@ module TmpFile
     end
   end
 
+  SLASH_REPLACE = '·'
   def self.tmp_for_file(file, tmp_options = {}, other_options = {})
-    tmp_for_file = IndiferentHash.process_options tmp_options, :file
+    tmp_for_file, prefix, key, persistence_dir = IndiferentHash.process_options tmp_options, :file, :prefix, :key, :dir
     return tmp_for_file unless tmp_for_file.nil?
 
-    prefix = IndiferentHash.process_options tmp_options, :prefix
-
     if prefix.nil?
-      perfile = file.to_s.gsub(/\//, '>') 
+      perfile = file.to_s.sub(/\.b?gz$/,'')
     else
-      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
+      perfile = prefix.to_s + ":" + file.to_s.sub(/\.b?gz$/,'')
     end
 
-    perfile.sub!(/\.b?gz$/,'')
+    perfile += "[#{ key }]" if key
 
     if other_options.include? :filters
       other_options[:filters].each do |match,value|
@@ -113,10 +112,10 @@ module TmpFile
       end
     end
 
-    persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir 
+    persistence_dir = TmpFile.tmpdir if persistence_dir.nil?
     Path.setup(persistence_dir) unless Path === persistence_dir
 
-    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    filename = perfile.gsub(/\s/,'_').gsub('/', SLASH_REPLACE)
     clean_options = other_options.dup
     clean_options.delete :unnamed
     clean_options.delete "unnamed"