scout-gear 6.0.0 → 7.2.0

data/lib/scout/open/stream.rb

@@ -1,5 +1,5 @@
 module Open
-  BLOCK_SIZE = 1024
+  BLOCK_SIZE = 1024 * 8
 
   class << self
     attr_accessor :sensible_write_lock_dir
@@ -31,13 +31,16 @@ module Open
         Thread.current.report_on_exception = false
         consume_stream(io, false, into, into_close)
       end
+
       io.threads.push(consumer_thread) if io.respond_to?(:threads)
+      Thread.pass until consumer_thread["name"]
+
       consumer_thread
     else
       if into
-        Log.medium "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
+        Log.low "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
       else
-        Log.medium "Consuming stream #{Log.fingerprint io}"
+        Log.low "Consuming stream #{Log.fingerprint io}"
       end
 
       begin
@@ -49,11 +52,8 @@ module Open
           into_path, into = into, File.open(into, 'w') 
         end
         
-        into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
-        io.sync = true
 
-        Log.high "started consuming stream #{Log.fingerprint io}"
         begin
           while c = io.readpartial(BLOCK_SIZE)
             into << c if into
@@ -67,16 +67,15 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        Log.high "Done consuming stream #{Log.fingerprint io} into #{into_path || into}"
         c
       rescue Aborted
-        Log.high "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
+        Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
-        Log.high "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
-        exception = io.stream_exception || $!
+        Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
+        exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         into_path = into if into_path.nil? && String === into
@@ -109,7 +108,7 @@ module Open
         Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
         Open.consume_stream content 
       else
-        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
+        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
         begin
 
@@ -120,7 +119,7 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              f.sync = true
+              #f.sync = true
               begin
                 while block = content.readpartial(BLOCK_SIZE)
                   f.write block
@@ -145,12 +144,12 @@ module Open
 
           Open.notify_write(path) 
         rescue Aborted
-          Log.medium "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.medium "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
@@ -219,16 +218,15 @@ module Open
 
       #parent_pid = Process.pid
       pid = Process.fork {
-        purge_pipes(sin)
-        sout.close
         begin
+          purge_pipes(sin)
+          sout.close
 
           yield sin
           sin.close if close and not sin.closed? 
 
         rescue Exception
           Log.exception $!
-          #Process.kill :INT, parent_pid
           Kernel.exit!(-1)
         end
         Kernel.exit! 0
@@ -242,18 +240,18 @@ module Open
       ConcurrentStream.setup sout, :pair => sin
 
       thread = Thread.new do 
-        Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-        Thread.current.report_on_exception = false
         begin
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
           
           yield sin
 
           sin.close if close and not sin.closed? and not sin.aborted?
         rescue Aborted
-          Log.medium "Aborted open_pipe: #{$!.message}"
+          Log.low "Aborted open_pipe: #{$!.message}"
           raise $!
         rescue Exception
-          Log.medium "Exception in open_pipe: #{$!.message}"
+          Log.low "Exception in open_pipe: #{$!.message}"
           begin
             sout.threads.delete(Thread.current)
             sout.pair = []
@@ -269,6 +267,7 @@ module Open
 
       sin.threads = [thread]
       sout.threads = [thread]
+      Thread.pass until thread["name"]
     end
 
     sout
@@ -287,8 +286,8 @@ module Open
 
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
-        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
         Thread.current.report_on_exception = false
+        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
         begin
@@ -317,7 +316,7 @@ module Open
         out_pipes.each do |sout|
           sout.abort if sout.respond_to? :abort
         end
-        Log.medium "Tee aborting #{Log.fingerprint stream}"
+        Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
       rescue Exception
         begin
@@ -332,7 +331,7 @@ module Open
           in_pipes.each do |sin|
             sin.close unless sin.closed?
           end
-          Log.medium "Tee exception #{Log.fingerprint stream}"
+          Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
           Log.exception $!
         ensure
@@ -344,11 +343,10 @@ module Open
       end
     end
 
-
     out_pipes.each do |sout|
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
     end
-    splitter_thread.wakeup until splitter_thread["name"]
+    Thread.pass until splitter_thread["name"]
 
     main_pipe = out_pipes.first
     main_pipe.autojoin = true
@@ -399,5 +397,39 @@ module Open
     end
     str
   end
+  
+  def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
+    Open.open_pipe do |sin|
+      line = stream.gets
+      while line =~ /^#{header_hash}/ do
+        sin.puts line
+        line = stream.gets
+      end
+
+      line_stream = Open.open_pipe do |line_stream_in|
+        line_stream_in.puts line
+        begin
+          Open.consume_stream(stream, false, line_stream_in)
+        rescue
+          raise $!
+        end
+      end
+
+      sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+
+      begin
+        Open.consume_stream(sorted, false, sin)
+      rescue
+        Log.exception $!
+        begin
+          sorted.raise($!) if sorted.respond_to? :raise
+          stream.raise($!) if stream.respond_to? :raise
+        ensure
+          raise $!
+        end
+      end
+    end
+  end
+
 
 end

data/lib/scout/path/find.rb

@@ -9,7 +9,8 @@ module Path
         break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
-          file =~ /(?:scout|rbbt)\/persist.rb/
+          file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /modules\/rbbt-util/
       end
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
@@ -136,15 +137,38 @@ module Path
     found
   end
 
+  def self.exists_file_or_alternatives(file)
+    return file if File.exist?(file) or File.directory?(file)
+    %w(gz bgz zip).each do |extension|
+      alt_file = file + '.' + extension
+      return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
+    end
+    nil
+  end
+
   def find(where = nil)
-    return self if located?
+    if located?
+      if File.exist?(self)
+        return self if located?
+      else
+        found = Path.exists_file_or_alternatives(self)
+        if found
+          return self.annotate(found)
+        else
+          return self if located?
+        end
+      end
+    end
+
     return find_all if where == 'all' || where == :all
+
     return follow(where) if where
 
     map_order.each do |map_name|
       found = follow(map_name, false)
 
-      return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
+      found = Path.exists_file_or_alternatives(found)
+      return annotate_found_where(found, map_name) if found
     end
 
     return follow(:default)

data/lib/scout/path/util.rb

@@ -1,4 +1,9 @@
 module Path
+  def no_method_missing
+    class << self
+      undef_method :method_missing
+    end
+  end
 
   def self.is_filename?(string, need_to_exists = true)
     return false if string.nil?
@@ -56,9 +61,7 @@ module Path
     end.flatten.uniq
   end
 
-  def no_method_missing
-    class << self
-      undef_method :method_missing
-    end
+  def set_extension(extension)
+    self.annotate(self + ".#{extension}")
   end
 end

data/lib/scout/persist/serialize.rb

@@ -93,22 +93,15 @@ module Persist
 
     Log.debug "Save #{Log.fingerprint type} on #{file}"
     if save_drivers[type]
-      Open.write(file, save_drivers[type].call(content))
-      return
-    end
-
-    if IO === content || StringIO === content
-      main, copy = Open.tee_stream_thread content
-      t = Thread.new do
-        Thread.current["name"] = "file saver: " + file
-        Open.sensible_write(file, main)
+      if save_drivers[type].arity == 1
+        return Open.sensible_write(file, save_drivers[type].call(content))
+      else
+        return save_drivers[type].call(file, content)
       end
-      ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
-    else
-      serialized = serialize(content, type)
-      Open.sensible_write(file, serialized, :force => true)
-      content
     end
+    serialized = serialize(content, type)
+    Open.sensible_write(file, serialized, :force => true)
+    return nil
   end
 
   def self.load(file, type = :serializer)

data/lib/scout/persist.rb

@@ -26,6 +26,7 @@ module Persist
 
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
+    return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
     update = options[:update] || persist_options[:update]
@@ -35,10 +36,29 @@ module Persist
     if Open.exist?(file) && ! update
       Persist.load(file, type)
     else
+      return yield(file) if block.arity == 1
       res = yield
       begin
         Open.rm(file)
-        res = Persist.save(res, file, type)
+
+        if IO === res || StringIO === res
+          tee_copies = options[:tee_copies] || 1
+          main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+          t = Thread.new do
+            Thread.current.report_on_exception = false
+            Thread.current["name"] = "file saver: " + file
+            Open.sensible_write(file, main)
+          end
+          Thread.pass until t["name"]
+          copies.each_with_index do |copy,i|
+            next_stream = copies[i+1] if copies.length > i
+            ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+          end
+          res = copies.first
+        else
+          pres = Persist.save(res, file, type)
+          res = pres unless pres.nil?
+        end
       rescue
         raise $! unless options[:canfail]
         Log.debug "Could not persist #{type} on #{file}"

data/lib/scout/resource/produce.rb

@@ -39,7 +39,7 @@ module Resource
       else
         ScoutRake.run(rakefile, rake_dir, task)
       end
-    rescue Rake::TaskNotFound
+    rescue ScoutRake::TaskNotFound
       if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
         raise $! 
       end
@@ -51,9 +51,9 @@ module Resource
 
   def produce(path, force = false)
     case
-    when @resources.include?(path)
+    when (@resources && @resources.include?(path))
       type, content = @resources[path]
-    when (Path === path && @resources.include?(path.original))
+    when (Path === path && @resources && @resources.include?(path.original))
       type, content = @resources[path.original]
     when has_rake(path)
       type = :rake
@@ -126,97 +126,10 @@ module Resource
             when :rake
               run_rake(path, content, rake_dir)
             when :install
-              Log.debug "Installing software: #{path}"
-
-              $set_software_env = false unless File.exist? path
-              
-              software_dir = path.resource.root.software.find :user
-              helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
-              #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
-
-              preamble = <<-EOF
-#!/bin/bash
-
-RBBT_SOFTWARE_DIR="#{software_dir}"
-
-INSTALL_HELPER_FILE="#{helper_file}"
-source "$INSTALL_HELPER_FILE"
-              EOF
-
-              content = content.call if Proc === content
-
-              content = if content =~ /git:|\.git$/
-                          {:git => content}
-                        else
-                          {:src => content}
-                        end if String === content and Open.remote?(content)
-
-              script_text = case content
-                            when nil
-                              raise "No way to install #{path}"
-                            when Path
-                              Open.read(content) 
-                            when String
-                              if Path.is_filename?(content) and Open.exists?(content)
-                                Open.read(content) 
-                              else
-                                content
-                              end
-                            when Hash
-                              name = content[:name] || File.basename(path)
-                              git = content[:git]
-                              src = content[:src]
-                              url = content[:url]
-                              jar = content[:jar]
-                              extra = content[:extra]
-                              commands = content[:commands]
-                              if git
-                                <<-EOF
-
-name='#{name}'
-url='#{git}'
-
-install_git "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif src
-                                <<-EOF
-
-name='#{name}'
-url='#{src}'
-
-install_src "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif jar
-                                <<-EOF
-
-name='#{name}'
-url='#{jar}'
-
-install_jar "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              else
-                                <<-EOF
-
-name='#{name}'
-url='#{url}'
-
-#{commands}
-                                EOF
-                              end
-                            end
-
-              script = preamble + "\n" + script_text
-              Log.debug "Installing software with script:\n" << script
-              CMD.cmd_log('bash', :in => script)
-
-              set_software_env(software_dir) unless $set_software_env
-              $set_software_env = true
+              software_dir = self.root.software
+              name = File.basename(path)
+              Resource.install(content, name, software_dir)
+              set_software_env(software_dir)
             else
               raise "Could not produce #{ resource }. (#{ type }, #{ content })"
             end

data/lib/scout/resource/software.rb

@@ -0,0 +1,176 @@
+module Resource
+
+  def self.install_helpers
+    File.expand_path(Scout.share.software.install_helpers.find(:lib))
+  end
+
+  def self.install(content, name, software_dir = Path.setup('software'), &block)
+    software_dir ||= Path.setup('software')
+    software_dir = software_dir.find if Path === software_dir
+
+    content = block if block_given?
+
+    preamble = <<-EOF
+#!/bin/bash
+
+SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{install_helpers}"
+source "$INSTALL_HELPER_FILE"
+    EOF
+
+    content = content.call if Proc === content
+
+    name = content[:name] if Hash === content && content.include?(:name)
+    content = 
+      if content =~ /git:|\.git$/
+        {:git => content}
+      else
+        {:src => content}
+      end if String === content and Open.remote?(content)
+
+      script_text = 
+        case content
+        when nil
+          raise "No way to install #{name}"
+        when Path
+          Open.read(content) 
+        when String
+          if Path.is_filename?(content) and Open.exists?(content)
+            Open.read(content) 
+          else
+            content
+          end
+        when Hash
+          name = content[:name] || name
+          git = content[:git]
+          src = content[:src]
+          url = content[:url]
+          jar = content[:jar]
+          extra = content[:extra]
+          commands = content[:commands]
+          if git
+            <<-EOF
+
+name='#{name}'
+url='#{git}'
+
+install_git "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif src
+            <<-EOF
+
+name='#{name}'
+url='#{src}'
+
+install_src "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif jar
+            <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          else
+            <<-EOF
+
+name='#{name}'
+url='#{url}'
+
+#{commands}
+            EOF
+          end
+        end
+
+      script = preamble + "\n" + script_text
+      Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
+      CMD.cmd_log('bash', :in => script)
+      Resource.set_software_env(software_dir)
+  end
+
+  def self.set_software_env(software_dir = Path.setup('software'))
+    software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
+      next unless software_dir.exists?
+      Log.medium "Preparing software env at #{software_dir}"
+
+      software_dir = File.expand_path(software_dir)
+      opt_dir = File.join(software_dir, 'opt')
+      bin_dir = File.join(opt_dir, 'bin')
+
+      Misc.env_add 'PATH', bin_dir
+
+      FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
+
+      %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
+        filename = File.join(opt_dir, file)
+        begin
+          FileUtils.touch filename unless File.exist? filename
+        rescue
+          Log.warn("Could not touch #{ filename }")
+        end
+      end
+
+      Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CPLUS_INCLUDE_PATH',dir)
+        Misc.env_add('C_INCLUDE_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.c-paths')
+
+      Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('LIBRARY_PATH',dir)
+        Misc.env_add('LD_LIBRARY_PATH',dir)
+        Misc.env_add('LD_RUN_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.ld-paths')
+
+      Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('PKG_CONFIG_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
+
+      Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
+      end if File.exist? File.join(opt_dir, '.aclocal-paths')
+
+      Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CLASSPATH', "#{dir}")
+      end if File.exist? File.join(opt_dir, '.java-classpaths')
+
+      Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
+        Misc.env_add('CLASSPATH', "#{file}")
+      end
+
+      if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
+        Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
+
+          # Load exports
+          Open.read(file).split("\n").each do |line|
+            next unless line =~ /^\s*export\s+([^=]+)=(.*)/
+            var = $1.strip
+            value = $2.strip
+            value.sub!(/^['"]/,'')
+            value.sub!(/['"]$/,'')
+            value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
+            Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
+            ENV[var] = value
+          end
+        end
+      end
+    end
+  end
+end