scout-gear 7.2.0 → 8.0.0

data/lib/scout/open/stream.rb

@@ -54,11 +54,10 @@ module Open
         
         into_close = false unless into.respond_to? :close
 
-        begin
-          while c = io.readpartial(BLOCK_SIZE)
-            into << c if into
-          end
-        rescue EOFError
+        while c = io.read(BLOCK_SIZE)
+          into << c if into
+          last_c = c if c
+          break if io.closed?
         end
 
         io.join if io.respond_to? :join
@@ -67,13 +66,15 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        c
+        last_c
       rescue Aborted
+        Thread.current["exception"] = true
         Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
+        Thread.current["exception"] = true
         Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
         exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
@@ -111,7 +112,6 @@ module Open
         FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
         begin
-
           case
           when block_given?
             File.open(tmp_path, 'wb', &block)
@@ -119,13 +119,10 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              #f.sync = true
-              begin
-                while block = content.readpartial(BLOCK_SIZE)
-                  f.write block
-                end 
-              rescue EOFError
-              end
+              while block = content.read(BLOCK_SIZE)
+                f.write block
+                break if content.closed?
+              end 
             end
           else
             File.open(tmp_path, 'wb') do |f|  end
@@ -143,14 +140,15 @@ module Open
           content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
 
           Open.notify_write(path) 
+          Log.debug "Done sensible write: [#{Process.pid}] -- #{ path }"
         rescue Aborted
-          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
-          content.abort if content.respond_to? :abort
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ path }"
+          content.abort(exception) if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
         rescue
@@ -191,7 +189,7 @@ module Open
       FileUtils.rm path if erase && File.exist?(path)
     end
   end
-  
+
   def self.release_pipes(*pipes)
     PIPE_MUTEX.synchronize do
       pipes.flatten.each do |pipe|
@@ -243,7 +241,7 @@ module Open
         begin
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-          
+
           yield sin
 
           sin.close if close and not sin.closed? and not sin.aborted?
@@ -290,31 +288,36 @@ module Open
         Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
-        begin
-          while block = stream.readpartial(BLOCK_SIZE)
-
-            in_pipes.each_with_index do |sin,i|
-              begin 
-                sin.write block
-              rescue IOError
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
-                skip[i] = true
-              rescue
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
-                raise $!
-              end unless skip[i] 
-            end
+        while block = stream.read(BLOCK_SIZE)
+
+          in_pipes.each_with_index do |sin,i|
+            begin 
+              sin.write block
+            rescue IOError
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
+              skip[i] = true
+            rescue
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
+              raise $!
+            end unless skip[i] 
           end
-        rescue IOError
+          break if stream.closed?
         end
 
         stream.join if stream.respond_to? :join
         stream.close unless stream.closed?
         in_pipes.first.close unless in_pipes.first.closed?
       rescue Aborted, Interrupt
-        stream.abort if stream.respond_to? :abort
-        out_pipes.each do |sout|
-          sout.abort if sout.respond_to? :abort
+        stream.abort if stream.respond_to?(:abort) && ! stream.aborted?
+        out_pipes.reverse.each do |sout|
+          sout.threads.delete(Thread.current)
+          begin
+            sout.abort($!) if sout.respond_to?(:abort) && ! sout.aborted?
+          rescue
+          end
+        end
+        in_pipes.each do |sin|
+          sin.close unless sin.closed?
         end
         Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
@@ -333,12 +336,14 @@ module Open
           end
           Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
-          Log.exception $!
         ensure
-          in_pipes.each do |sin|
-            sin.close unless sin.closed?
+          begin
+            in_pipes.each do |sin|
+              sin.close unless sin.closed?
+            end
+          ensure
+            raise $!
           end
-          raise $!
         end
       end
     end
@@ -397,7 +402,7 @@ module Open
     end
     str
   end
-  
+
   def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
     Open.open_pipe do |sin|
       line = stream.gets
@@ -431,5 +436,69 @@ module Open
     end
   end
 
+  def self.process_stream(s)
+    begin
+      yield s
+      s.close if s.respond_to?(:close) && ! s.closed?
+      s.join if s.respond_to?(:join)
+    rescue
+      s.abort($!) if s.respond_to? :abort
+      raise $!
+    end
+  end
+
+
+  def self.collapse_stream(s, line: nil, sep: "\t", header: nil, &block)
+    sep ||= "\t"
+    Open.open_pipe do |sin|
+      sin.puts header if header
+      process_stream(s) do |s|
+        line ||= s.gets
+
+        current_parts = []
+        while line 
+          key, *parts = line.chomp.split(sep, -1)
+          case
+          when key.nil?
+          when current_parts.nil?
+            current_parts = parts
+            current_key = key
+          when current_key == key
+            parts.each_with_index do |part,i|
+              if current_parts[i].nil?
+                current_parts[i] = "|" << part
+              else
+                current_parts[i] = current_parts[i] << "|" << part
+              end
+            end
+
+            (parts.length..current_parts.length-1).to_a.each do |pos|
+              current_parts[pos] = current_parts[pos] << "|" << ""
+            end
+          when current_key.nil?
+            current_key = key
+            current_parts = parts
+          when current_key != key
+            if block_given?
+              res = block.call(current_parts)
+              sin.puts [current_key, res] * sep
+            else
+              sin.puts [current_key, current_parts].flatten * sep
+            end 
+            current_key = key
+            current_parts = parts
+          end
+          line = s.gets
+        end
+
+        if block_given?
+          res = block.call(current_parts)
+          sin.puts [current_key, res] * sep
+        else
+          sin.puts [current_key, current_parts].flatten * sep
+        end unless current_key.nil?
+      end
+    end
+  end
 
 end

data/lib/scout/open/util.rb

@@ -94,6 +94,11 @@ module Open
     File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
+  def self.directory?(file)
+    file = file.find if Path === file
+    File.directory?(file)
+  end
+
   def self.exists?(file)
     file = file.find if Path === file
     File.exist?(file)
@@ -111,7 +116,7 @@ module Open
   end
 
   def self.rm(file)
-    FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
+    FileUtils.rm(file) if File.exist?(file) || Open.broken_link?(file)
   end
 
   def self.rm_rf(file)
@@ -155,7 +160,7 @@ module Open
     begin
       if File.symlink?(file) || File.stat(file).nlink > 1
         if File.exist?(file + '.info') && defined?(Step)
-          done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
+          done = Persist.load(file + '.info', Step::SERIALIZER)[:done]
           return done if done
         end
 
@@ -173,7 +178,7 @@ module Open
     target = target.find if Path === target
 
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
-    FileUtils.rm target if File.exist?(target)
+    FileUtils.rm_rf target if File.exist?(target)
     FileUtils.cp_r source, target
   end
 
@@ -222,4 +227,9 @@ module Open
     end
     nil
   end
+
+  def self.list(file)
+    file = file.produce_and_find if Path === file
+    Open.read(file).split("\n")
+  end
 end

data/lib/scout/path/find.rb

@@ -10,8 +10,10 @@ module Path
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
           file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /scout\/resource\/produce.rb/ or
           file =~ /modules\/rbbt-util/
       end
+      return nil if file.nil?
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
 
@@ -38,7 +40,7 @@ module Path
       sub('{SUBPATH}', path._subpath).
       sub('{BASENAME}', File.basename(path)).
       sub('{PATH}', path).
-      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir).
+      sub('{LIBDIR}', path.libdir || (path.pkgdir.respond_to?(:libdir) && path.pkgdir.libdir) || Path.caller_lib_dir || "NOLIBDIR").
       sub('{MAPNAME}', map_name.to_s).
       sub('{REMOVE}/', '').
       sub('{REMOVE}', '').gsub(/\/+/,'/')
@@ -188,4 +190,10 @@ module Path
       .select{|file| file.exist? }.uniq
   end
 
+  def find_with_extension(extension, *args)
+    found = self.find(*args)
+    return found if found.exists?
+    found_with_extension = self.set_extension(extension).find
+    found_with_extension.exists? ? found_with_extension : found
+  end
 end

data/lib/scout/path/util.rb

@@ -12,6 +12,23 @@ module Path
     return false
   end
 
+  def self.sanitize_filename(filename, length = 254)
+    if filename.length > length
+      if filename =~ /(\..{2,9})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
   def directory?
     return nil unless self.exist?
     File.directory?(self.find)
@@ -64,4 +81,22 @@ module Path
   def set_extension(extension)
     self.annotate(self + ".#{extension}")
   end
+
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
+    return true if not Open.exists?(file)
+    path = path.find if Path === path
+    file = file.find if Path === file
+    if by_link
+      patht = File.exist?(path) ? File.lstat(path).mtime : nil
+      filet = File.exist?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
+    return true if patht.nil? || filet.nil?
+    diff = patht - filet
+    return diff if diff < 0
+    return false
+  end
 end

data/lib/scout/persist/serialize.rb

@@ -1,5 +1,6 @@
 require_relative '../open'
 require_relative 'open'
+require 'set'
 
 module Persist
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
@@ -19,7 +20,7 @@ module Persist
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
     case type
-    when nil, :string, :integer, :float, :boolean, :file, :path
+    when nil, :string, :integer, :float, :boolean, :file, :path, :select, :folder
       if IO === content || StringIO === content
         content.read
       else
@@ -46,8 +47,9 @@ module Persist
   def self.deserialize(serialized, type)
     type = type.to_sym if String === type
     type = SERIALIZER if type == :serializer
+
     case type
-    when nil, :string, :file, :stream
+    when nil, :string, :file, :stream, :select, :folder
       serialized
     when :path
       Path.setup(serialized)
@@ -99,9 +101,18 @@ module Persist
         return save_drivers[type].call(file, content)
       end
     end
-    serialized = serialize(content, type)
-    Open.sensible_write(file, serialized, :force => true)
-    return nil
+
+    if type == :binary
+      content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
+      Open.open(path, :mode => 'wb') do |f|
+        f.puts content
+      end
+      content
+   else
+      serialized = serialize(content, type)
+      Open.sensible_write(file, serialized, :force => true)
+      return nil
+    end
   end
 
   def self.load(file, type = :serializer)
@@ -118,6 +129,8 @@ module Persist
     end
 
     case type
+    when :binary
+      Open.read(file, :mode => 'rb')
     when :yaml
       Open.yaml(file)
     when :json

data/lib/scout/persist.rb

@@ -14,61 +14,91 @@ module Persist
 
     attr_writer :lock_dir
     def lock_dir
-      @lock_dir ||= Path.setup("var/cache/persist_locks")
+      @lock_dir ||= Path.setup("tmp/persist_locks").find
     end
   end
 
   def self.persistence_path(name, options = {})
     options = IndiferentHash.add_defaults options, :dir => Persist.cache_dir
     other_options = IndiferentHash.pull_keys options, :other
-    TmpFile.tmp_for_file(name, options, other_options)
+    name = name.filename if name.respond_to?(:filename) && name.filename
+    persist_options = {}
+    TmpFile.tmp_for_file(name, options.merge(persist_options), other_options)
   end
 
+  MEMORY_CACHE = {}
+  CONNECTIONS = {}
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
     return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
+    if type == :memory
+      repo = options[:memory] || options[:repo] || MEMORY_CACHE
+      repo[file] ||= yield
+      return repo[file]
+    end
+
     update = options[:update] || persist_options[:update]
     update = Open.mtime(update) if Path === update
     update = Open.mtime(file) >= update ? false : true if Time === update
 
-    if Open.exist?(file) && ! update
-      Persist.load(file, type)
-    else
-      return yield(file) if block.arity == 1
-      res = yield
-      begin
-        Open.rm(file)
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
+    Open.lock lockfile do |lock|
+      if Open.exist?(file) && ! update
+        Persist.load(file, type)
+      else
+        begin
+          file = file.find if Path === file
+          return yield(file) if block.arity == 1
+          res = yield
+
+          if res.nil?
+            return Persist.load(file, type)
+          end
+
+          Open.rm(file)
 
-        if IO === res || StringIO === res
-          tee_copies = options[:tee_copies] || 1
-          main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
-          t = Thread.new do
-            Thread.current.report_on_exception = false
-            Thread.current["name"] = "file saver: " + file
-            Open.sensible_write(file, main)
+          if IO === res || StringIO === res
+            tee_copies = options[:tee_copies] || 1
+            main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+            main.lock = lock
+            t = Thread.new do
+              Thread.current.report_on_exception = false
+              Thread.current["name"] = "file saver: " + file
+              Open.sensible_write(file, main)
+            end
+            Thread.pass until t["name"]
+            copies.each_with_index do |copy,i|
+              next_stream = copies[i+1] if copies.length > i
+              ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+            end
+            res = copies.first
+            raise KeepLocked.new(res)
+          else
+            pres = Persist.save(res, file, type)
+            res = pres unless pres.nil?
           end
-          Thread.pass until t["name"]
-          copies.each_with_index do |copy,i|
-            next_stream = copies[i+1] if copies.length > i
-            ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+        rescue
+          Thread.handle_interrupt(Exception => :never) do
+            if Open.exist?(file)
+              Log.debug "Failed persistence #{file} - erasing"
+              Open.rm file
+            else
+              Log.debug "Failed persistence #{file}"
+            end
           end
-          res = copies.first
-        else
-          pres = Persist.save(res, file, type)
-          res = pres unless pres.nil?
+          raise $! unless options[:canfail]
         end
-      rescue
-        raise $! unless options[:canfail]
-        Log.debug "Could not persist #{type} on #{file}"
+        res
       end
-      res
     end
   end
 
-  def self.memory(name, *args, &block)
-    self.persist(name, :memory, *args, &block)
+  def self.memory(name, options = {}, &block)
+    options[:persist_path] ||= options[:path] ||= [name, options[:key]].compact * ":"
+    self.persist(name, :memory, options, &block)
   end
 
 end

data/lib/scout/resource/path.rb

@@ -1,4 +1,47 @@
 module Path
+  def produce(force = false)
+    return self if ! force && (Open.exist?(self) || @produced)
+    begin
+      if Resource === self.pkgdir
+        self.pkgdir.produce self
+      else
+        false
+      end
+    rescue ResourceNotFound
+      false
+    rescue
+      message = $!.message
+      message = "No exception message" if message.nil? || message.empty?
+      Log.warn "Error producing #{self}: #{message}"
+      false
+    ensure
+      @produced = true
+    end
+  end
+
+  def produce_with_extension(extension, *args)
+    begin
+      self.produce(*args)
+    rescue Exception
+      exception = $!
+      begin
+        self.set_extension(extension).produce(*args)
+      rescue Exception
+        raise exception
+      end
+    end
+  end
+
+  def produce_and_find(extension = nil, *args)
+    if extension
+      found = find_with_extension(extension, *args)
+      found.exists? ? found : produce_with_extension(extension, *args)
+    else
+      found = find
+      found.exists? ? found : produce(*args)
+    end
+  end
+
   def relocate
     return self if Open.exists?(self)
     Resource.relocate(self)
@@ -17,4 +60,14 @@ module Path
   def write(*args, &block)
     Open.write(self.find, *args, &block)
   end
+
+  def list
+    found = produce_and_find('list')
+    Open.list(found)
+  end
+
+  def exists?
+    return true if Open.exists?(self.find)
+    self.produce
+  end
 end

data/lib/scout/resource/produce.rb

@@ -149,11 +149,3 @@ module Resource
   end
 
 end
-
-module Path
-  def produce(force = false)
-    return self if ! force && Open.exist?(self)
-    self.pkgdir.produce self if Resource === self.pkgdir
-    return self
-  end
-end

data/lib/scout/resource/util.rb

@@ -1,7 +1,8 @@
 module Resource
   def identify(path)
     return path unless path.start_with?("/")
-    path_maps = path.path_maps || self.path_maps || Path.path_maps
+    path_maps = path.path_maps if Path === path
+    path_maps ||= self.path_maps || Path.path_maps
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
 

data/lib/scout/semaphore.rb

@@ -37,7 +37,13 @@ if continue
     int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
+    if (sem == SEM_FAILED){
+      return(errno);
+    }
     ret = sem_wait(sem);
+    if (ret == -1){
+      return(errno);
+    }
     sem_close(sem);
     return(ret);
   }
@@ -51,6 +57,7 @@ if continue
     sem_close(sem);
   }
       EOF
+
     end
 
     SEM_MUTEX = Mutex.new
@@ -66,7 +73,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end