scout-gear 7.1.0 → 7.3.0

data/lib/scout/named_array.rb

@@ -0,0 +1,68 @@
+require_relative 'meta_extension'
+module NamedArray
+  extend MetaExtension
+  extension_attr :fields
+
+  def self.identify_name(names, selected)
+    res = (Array === selected ? selected : [selected]).collect do |field|
+      case field
+      when nil
+        0
+      when Integer
+        field
+      when Symbol
+        field == :key ? field : identify_name(names, field.to_s)
+      when (names.nil? and String)
+        if field =~ /^\d+$/
+          identify_field(key_field, fields, field.to_i)
+        else
+          raise "No name information available and specified name not numeric: #{ field }"
+        end
+      when Symbol
+        names.index{|f| f.to_s == field.to_s }
+      when String
+        pos = names.index{|f| f.to_s == field }
+        next pos if pos
+        if field =~ /^\d+$/
+          next identify_names(names, field.to_i)
+        end
+        pos = names.index{|name| name.start_with?(field) }
+        next pos if pos
+        nil
+      else
+        raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
+      end
+    end
+
+    Array === selected ? res : res.first
+  end
+
+  def positions(fields)
+    if Array ==  fields
+      fields.collect{|field|
+        NamedArray.identify_name(@fields, field)
+      }
+    else
+      NamedArray.identify_name(@fields, fields)
+    end
+  end
+
+  def [](key)
+    pos = NamedArray.identify_name(@fields, key)
+    super(pos)
+  end
+
+  def concat(other)
+    super(other)
+    self.fields.concat(other.fields) if NamedArray === other
+    self
+  end
+
+  def to_hash
+    hash = {}
+    self.fields.zip(self) do |field,value|
+      hash[field] = value
+    end
+    IndiferentHash.setup hash
+  end
+end

data/lib/scout/open/stream.rb

@@ -1,5 +1,5 @@
 module Open
-  BLOCK_SIZE = 1024
+  BLOCK_SIZE = 1024 * 8
 
   class << self
     attr_accessor :sensible_write_lock_dir
@@ -52,15 +52,11 @@ module Open
           into_path, into = into, File.open(into, 'w') 
         end
         
-        into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
-        io.sync = true
 
-        begin
-          while c = io.readpartial(BLOCK_SIZE)
-            into << c if into
-          end
-        rescue EOFError
+        while c = io.read(BLOCK_SIZE)
+          into << c if into
+          break if io.closed?
         end
 
         io.join if io.respond_to? :join
@@ -77,7 +73,7 @@ module Open
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
         Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
-        exception = io.stream_exception || $!
+        exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         into_path = into if into_path.nil? && String === into
@@ -110,7 +106,7 @@ module Open
         Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
         Open.consume_stream content 
       else
-        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
+        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
         begin
 
@@ -121,13 +117,11 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              f.sync = true
-              begin
-                while block = content.readpartial(BLOCK_SIZE)
-                  f.write block
-                end 
-              rescue EOFError
-              end
+              #f.sync = true
+              while block = content.read(BLOCK_SIZE)
+                f.write block
+                break if content.closed?
+              end 
             end
           else
             File.open(tmp_path, 'wb') do |f|  end
@@ -292,22 +286,20 @@ module Open
         Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
-        begin
-          while block = stream.readpartial(BLOCK_SIZE)
-
-            in_pipes.each_with_index do |sin,i|
-              begin 
-                sin.write block
-              rescue IOError
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
-                skip[i] = true
-              rescue
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
-                raise $!
-              end unless skip[i] 
-            end
+        while block = stream.read(BLOCK_SIZE)
+
+          in_pipes.each_with_index do |sin,i|
+            begin 
+              sin.write block
+            rescue IOError
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
+              skip[i] = true
+            rescue
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
+              raise $!
+            end unless skip[i] 
           end
-        rescue IOError
+          break if stream.closed?
         end
 
         stream.join if stream.respond_to? :join
@@ -345,7 +337,6 @@ module Open
       end
     end
 
-
     out_pipes.each do |sout|
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
     end
@@ -401,4 +392,38 @@ module Open
     str
   end
 
+  def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
+    Open.open_pipe do |sin|
+      line = stream.gets
+      while line =~ /^#{header_hash}/ do
+        sin.puts line
+        line = stream.gets
+      end
+
+      line_stream = Open.open_pipe do |line_stream_in|
+        line_stream_in.puts line
+        begin
+          Open.consume_stream(stream, false, line_stream_in)
+        rescue
+          raise $!
+        end
+      end
+
+      sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+
+      begin
+        Open.consume_stream(sorted, false, sin)
+      rescue
+        Log.exception $!
+        begin
+          sorted.raise($!) if sorted.respond_to? :raise
+          stream.raise($!) if stream.respond_to? :raise
+        ensure
+          raise $!
+        end
+      end
+    end
+  end
+
+
 end

data/lib/scout/path/find.rb

@@ -9,7 +9,8 @@ module Path
         break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
-          file =~ /(?:scout|rbbt)\/persist.rb/
+          file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /modules\/rbbt-util/
       end
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
@@ -136,15 +137,38 @@ module Path
     found
   end
 
+  def self.exists_file_or_alternatives(file)
+    return file if File.exist?(file) or File.directory?(file)
+    %w(gz bgz zip).each do |extension|
+      alt_file = file + '.' + extension
+      return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
+    end
+    nil
+  end
+
   def find(where = nil)
-    return self if located?
+    if located?
+      if File.exist?(self)
+        return self if located?
+      else
+        found = Path.exists_file_or_alternatives(self)
+        if found
+          return self.annotate(found)
+        else
+          return self if located?
+        end
+      end
+    end
+
     return find_all if where == 'all' || where == :all
+
     return follow(where) if where
 
     map_order.each do |map_name|
       found = follow(map_name, false)
 
-      return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
+      found = Path.exists_file_or_alternatives(found)
+      return annotate_found_where(found, map_name) if found
     end
 
     return follow(:default)

data/lib/scout/path/util.rb

@@ -1,4 +1,9 @@
 module Path
+  def no_method_missing
+    class << self
+      undef_method :method_missing
+    end
+  end
 
   def self.is_filename?(string, need_to_exists = true)
     return false if string.nil?
@@ -56,9 +61,7 @@ module Path
     end.flatten.uniq
   end
 
-  def no_method_missing
-    class << self
-      undef_method :method_missing
-    end
+  def set_extension(extension)
+    self.annotate(self + ".#{extension}")
   end
 end

data/lib/scout/persist/serialize.rb

@@ -93,22 +93,15 @@ module Persist
 
     Log.debug "Save #{Log.fingerprint type} on #{file}"
     if save_drivers[type]
-      Open.write(file, save_drivers[type].call(content))
-      return
-    end
-
-    if IO === content || StringIO === content
-      main, copy = Open.tee_stream_thread content
-      t = Thread.new do
-        Thread.current["name"] = "file saver: " + file
-        Open.sensible_write(file, main)
+      if save_drivers[type].arity == 1
+        return Open.sensible_write(file, save_drivers[type].call(content))
+      else
+        return save_drivers[type].call(file, content)
       end
-      ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
-    else
-      serialized = serialize(content, type)
-      Open.sensible_write(file, serialized, :force => true)
-      content
     end
+    serialized = serialize(content, type)
+    Open.sensible_write(file, serialized, :force => true)
+    return nil
   end
 
   def self.load(file, type = :serializer)

data/lib/scout/persist.rb

@@ -14,7 +14,7 @@ module Persist
 
     attr_writer :lock_dir
     def lock_dir
-      @lock_dir ||= Path.setup("var/cache/persist_locks")
+      @lock_dir ||= Path.setup("tmp/persist_locks").find
     end
   end
 
@@ -24,26 +24,60 @@ module Persist
     TmpFile.tmp_for_file(name, options, other_options)
   end
 
+  MEMORY_CACHE = {}
+  CONNECTIONS = {}
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
+    return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
     update = options[:update] || persist_options[:update]
     update = Open.mtime(update) if Path === update
     update = Open.mtime(file) >= update ? false : true if Time === update
 
-    if Open.exist?(file) && ! update
-      Persist.load(file, type)
-    else
-      res = yield
-      begin
-        Open.rm(file)
-        res = Persist.save(res, file, type)
-      rescue
-        raise $! unless options[:canfail]
-        Log.debug "Could not persist #{type} on #{file}"
+    if type == :memory
+      repo = options[:memory] || options[:repo] || MEMORY_CACHE
+      repo[file] ||= yield
+      return repo[file]
+    end
+
+    Open.lock lockfile do |lock|
+      if Open.exist?(file) && ! update
+        Persist.load(file, type)
+      else
+        return yield(file) if block.arity == 1
+        res = yield
+        begin
+          Open.rm(file)
+
+          if IO === res || StringIO === res
+            tee_copies = options[:tee_copies] || 1
+            main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+            main.lock = lock
+            t = Thread.new do
+              Thread.current.report_on_exception = false
+              Thread.current["name"] = "file saver: " + file
+              Open.sensible_write(file, main)
+            end
+            Thread.pass until t["name"]
+            copies.each_with_index do |copy,i|
+              next_stream = copies[i+1] if copies.length > i
+              ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+            end
+            res = copies.first
+            raise KeepLocked.new(res)
+          else
+            pres = Persist.save(res, file, type)
+            res = pres unless pres.nil?
+          end
+        rescue
+          raise $! unless options[:canfail]
+          Log.debug "Could not persist #{type} on #{file}"
+        end
+        res
       end
-      res
     end
   end
 

data/lib/scout/resource/produce.rb

@@ -39,7 +39,7 @@ module Resource
       else
         ScoutRake.run(rakefile, rake_dir, task)
       end
-    rescue Rake::TaskNotFound
+    rescue ScoutRake::TaskNotFound
       if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
         raise $! 
       end
@@ -51,9 +51,9 @@ module Resource
 
   def produce(path, force = false)
     case
-    when @resources.include?(path)
+    when (@resources && @resources.include?(path))
       type, content = @resources[path]
-    when (Path === path && @resources.include?(path.original))
+    when (Path === path && @resources && @resources.include?(path.original))
       type, content = @resources[path.original]
     when has_rake(path)
       type = :rake
@@ -126,97 +126,10 @@ module Resource
             when :rake
               run_rake(path, content, rake_dir)
             when :install
-              Log.debug "Installing software: #{path}"
-
-              $set_software_env = false unless File.exist? path
-              
-              software_dir = path.resource.root.software.find :user
-              helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
-              #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
-
-              preamble = <<-EOF
-#!/bin/bash
-
-RBBT_SOFTWARE_DIR="#{software_dir}"
-
-INSTALL_HELPER_FILE="#{helper_file}"
-source "$INSTALL_HELPER_FILE"
-              EOF
-
-              content = content.call if Proc === content
-
-              content = if content =~ /git:|\.git$/
-                          {:git => content}
-                        else
-                          {:src => content}
-                        end if String === content and Open.remote?(content)
-
-              script_text = case content
-                            when nil
-                              raise "No way to install #{path}"
-                            when Path
-                              Open.read(content) 
-                            when String
-                              if Path.is_filename?(content) and Open.exists?(content)
-                                Open.read(content) 
-                              else
-                                content
-                              end
-                            when Hash
-                              name = content[:name] || File.basename(path)
-                              git = content[:git]
-                              src = content[:src]
-                              url = content[:url]
-                              jar = content[:jar]
-                              extra = content[:extra]
-                              commands = content[:commands]
-                              if git
-                                <<-EOF
-
-name='#{name}'
-url='#{git}'
-
-install_git "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif src
-                                <<-EOF
-
-name='#{name}'
-url='#{src}'
-
-install_src "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif jar
-                                <<-EOF
-
-name='#{name}'
-url='#{jar}'
-
-install_jar "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              else
-                                <<-EOF
-
-name='#{name}'
-url='#{url}'
-
-#{commands}
-                                EOF
-                              end
-                            end
-
-              script = preamble + "\n" + script_text
-              Log.debug "Installing software with script:\n" << script
-              CMD.cmd_log('bash', :in => script)
-
-              set_software_env(software_dir) unless $set_software_env
-              $set_software_env = true
+              software_dir = self.root.software
+              name = File.basename(path)
+              Resource.install(content, name, software_dir)
+              set_software_env(software_dir)
             else
               raise "Could not produce #{ resource }. (#{ type }, #{ content })"
             end

data/lib/scout/resource/software.rb

@@ -0,0 +1,176 @@
+module Resource
+
+  def self.install_helpers
+    File.expand_path(Scout.share.software.install_helpers.find(:lib))
+  end
+
+  def self.install(content, name, software_dir = Path.setup('software'), &block)
+    software_dir ||= Path.setup('software')
+    software_dir = software_dir.find if Path === software_dir
+
+    content = block if block_given?
+
+    preamble = <<-EOF
+#!/bin/bash
+
+SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{install_helpers}"
+source "$INSTALL_HELPER_FILE"
+    EOF
+
+    content = content.call if Proc === content
+
+    name = content[:name] if Hash === content && content.include?(:name)
+    content = 
+      if content =~ /git:|\.git$/
+        {:git => content}
+      else
+        {:src => content}
+      end if String === content and Open.remote?(content)
+
+      script_text = 
+        case content
+        when nil
+          raise "No way to install #{name}"
+        when Path
+          Open.read(content) 
+        when String
+          if Path.is_filename?(content) and Open.exists?(content)
+            Open.read(content) 
+          else
+            content
+          end
+        when Hash
+          name = content[:name] || name
+          git = content[:git]
+          src = content[:src]
+          url = content[:url]
+          jar = content[:jar]
+          extra = content[:extra]
+          commands = content[:commands]
+          if git
+            <<-EOF
+
+name='#{name}'
+url='#{git}'
+
+install_git "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif src
+            <<-EOF
+
+name='#{name}'
+url='#{src}'
+
+install_src "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif jar
+            <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          else
+            <<-EOF
+
+name='#{name}'
+url='#{url}'
+
+#{commands}
+            EOF
+          end
+        end
+
+      script = preamble + "\n" + script_text
+      Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
+      CMD.cmd_log('bash', :in => script)
+      Resource.set_software_env(software_dir)
+  end
+
+  def self.set_software_env(software_dir = Path.setup('software'))
+    software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
+      next unless software_dir.exists?
+      Log.medium "Preparing software env at #{software_dir}"
+
+      software_dir = File.expand_path(software_dir)
+      opt_dir = File.join(software_dir, 'opt')
+      bin_dir = File.join(opt_dir, 'bin')
+
+      Misc.env_add 'PATH', bin_dir
+
+      FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
+
+      %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
+        filename = File.join(opt_dir, file)
+        begin
+          FileUtils.touch filename unless File.exist? filename
+        rescue
+          Log.warn("Could not touch #{ filename }")
+        end
+      end
+
+      Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CPLUS_INCLUDE_PATH',dir)
+        Misc.env_add('C_INCLUDE_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.c-paths')
+
+      Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('LIBRARY_PATH',dir)
+        Misc.env_add('LD_LIBRARY_PATH',dir)
+        Misc.env_add('LD_RUN_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.ld-paths')
+
+      Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('PKG_CONFIG_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
+
+      Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
+      end if File.exist? File.join(opt_dir, '.aclocal-paths')
+
+      Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CLASSPATH', "#{dir}")
+      end if File.exist? File.join(opt_dir, '.java-classpaths')
+
+      Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
+        Misc.env_add('CLASSPATH', "#{file}")
+      end
+
+      if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
+        Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
+
+          # Load exports
+          Open.read(file).split("\n").each do |line|
+            next unless line =~ /^\s*export\s+([^=]+)=(.*)/
+            var = $1.strip
+            value = $2.strip
+            value.sub!(/^['"]/,'')
+            value.sub!(/['"]$/,'')
+            value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
+            Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
+            ENV[var] = value
+          end
+        end
+      end
+    end
+  end
+end