scout-gear 7.1.0 → 7.2.0

data/lib/scout/path/util.rb

@@ -1,4 +1,9 @@
 module Path
+  def no_method_missing
+    class << self
+      undef_method :method_missing
+    end
+  end
 
   def self.is_filename?(string, need_to_exists = true)
     return false if string.nil?
@@ -56,9 +61,7 @@ module Path
     end.flatten.uniq
   end
 
-  def no_method_missing
-    class << self
-      undef_method :method_missing
-    end
+  def set_extension(extension)
+    self.annotate(self + ".#{extension}")
   end
 end

data/lib/scout/persist/serialize.rb

@@ -93,22 +93,15 @@ module Persist
 
     Log.debug "Save #{Log.fingerprint type} on #{file}"
     if save_drivers[type]
-      Open.write(file, save_drivers[type].call(content))
-      return
-    end
-
-    if IO === content || StringIO === content
-      main, copy = Open.tee_stream_thread content
-      t = Thread.new do
-        Thread.current["name"] = "file saver: " + file
-        Open.sensible_write(file, main)
+      if save_drivers[type].arity == 1
+        return Open.sensible_write(file, save_drivers[type].call(content))
+      else
+        return save_drivers[type].call(file, content)
       end
-      ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true
-    else
-      serialized = serialize(content, type)
-      Open.sensible_write(file, serialized, :force => true)
-      content
     end
+    serialized = serialize(content, type)
+    Open.sensible_write(file, serialized, :force => true)
+    return nil
   end
 
   def self.load(file, type = :serializer)

data/lib/scout/persist.rb

@@ -26,6 +26,7 @@ module Persist
 
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
+    return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
     update = options[:update] || persist_options[:update]
@@ -35,10 +36,29 @@ module Persist
     if Open.exist?(file) && ! update
       Persist.load(file, type)
     else
+      return yield(file) if block.arity == 1
       res = yield
       begin
         Open.rm(file)
-        res = Persist.save(res, file, type)
+
+        if IO === res || StringIO === res
+          tee_copies = options[:tee_copies] || 1
+          main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+          t = Thread.new do
+            Thread.current.report_on_exception = false
+            Thread.current["name"] = "file saver: " + file
+            Open.sensible_write(file, main)
+          end
+          Thread.pass until t["name"]
+          copies.each_with_index do |copy,i|
+            next_stream = copies[i+1] if copies.length > i
+            ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+          end
+          res = copies.first
+        else
+          pres = Persist.save(res, file, type)
+          res = pres unless pres.nil?
+        end
       rescue
         raise $! unless options[:canfail]
         Log.debug "Could not persist #{type} on #{file}"

data/lib/scout/resource/produce.rb

@@ -39,7 +39,7 @@ module Resource
       else
         ScoutRake.run(rakefile, rake_dir, task)
       end
-    rescue Rake::TaskNotFound
+    rescue ScoutRake::TaskNotFound
       if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
         raise $! 
       end
@@ -51,9 +51,9 @@ module Resource
 
   def produce(path, force = false)
     case
-    when @resources.include?(path)
+    when (@resources && @resources.include?(path))
       type, content = @resources[path]
-    when (Path === path && @resources.include?(path.original))
+    when (Path === path && @resources && @resources.include?(path.original))
       type, content = @resources[path.original]
     when has_rake(path)
       type = :rake
@@ -126,97 +126,10 @@ module Resource
             when :rake
               run_rake(path, content, rake_dir)
             when :install
-              Log.debug "Installing software: #{path}"
-
-              $set_software_env = false unless File.exist? path
-              
-              software_dir = path.resource.root.software.find :user
-              helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
-              #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
-
-              preamble = <<-EOF
-#!/bin/bash
-
-RBBT_SOFTWARE_DIR="#{software_dir}"
-
-INSTALL_HELPER_FILE="#{helper_file}"
-source "$INSTALL_HELPER_FILE"
-              EOF
-
-              content = content.call if Proc === content
-
-              content = if content =~ /git:|\.git$/
-                          {:git => content}
-                        else
-                          {:src => content}
-                        end if String === content and Open.remote?(content)
-
-              script_text = case content
-                            when nil
-                              raise "No way to install #{path}"
-                            when Path
-                              Open.read(content) 
-                            when String
-                              if Path.is_filename?(content) and Open.exists?(content)
-                                Open.read(content) 
-                              else
-                                content
-                              end
-                            when Hash
-                              name = content[:name] || File.basename(path)
-                              git = content[:git]
-                              src = content[:src]
-                              url = content[:url]
-                              jar = content[:jar]
-                              extra = content[:extra]
-                              commands = content[:commands]
-                              if git
-                                <<-EOF
-
-name='#{name}'
-url='#{git}'
-
-install_git "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif src
-                                <<-EOF
-
-name='#{name}'
-url='#{src}'
-
-install_src "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              elsif jar
-                                <<-EOF
-
-name='#{name}'
-url='#{jar}'
-
-install_jar "$name" "$url" #{extra}
-
-#{commands}
-                                EOF
-                              else
-                                <<-EOF
-
-name='#{name}'
-url='#{url}'
-
-#{commands}
-                                EOF
-                              end
-                            end
-
-              script = preamble + "\n" + script_text
-              Log.debug "Installing software with script:\n" << script
-              CMD.cmd_log('bash', :in => script)
-
-              set_software_env(software_dir) unless $set_software_env
-              $set_software_env = true
+              software_dir = self.root.software
+              name = File.basename(path)
+              Resource.install(content, name, software_dir)
+              set_software_env(software_dir)
             else
               raise "Could not produce #{ resource }. (#{ type }, #{ content })"
             end

data/lib/scout/resource/software.rb

@@ -0,0 +1,176 @@
+module Resource
+
+  def self.install_helpers
+    File.expand_path(Scout.share.software.install_helpers.find(:lib))
+  end
+
+  def self.install(content, name, software_dir = Path.setup('software'), &block)
+    software_dir ||= Path.setup('software')
+    software_dir = software_dir.find if Path === software_dir
+
+    content = block if block_given?
+
+    preamble = <<-EOF
+#!/bin/bash
+
+SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{install_helpers}"
+source "$INSTALL_HELPER_FILE"
+    EOF
+
+    content = content.call if Proc === content
+
+    name = content[:name] if Hash === content && content.include?(:name)
+    content = 
+      if content =~ /git:|\.git$/
+        {:git => content}
+      else
+        {:src => content}
+      end if String === content and Open.remote?(content)
+
+      script_text = 
+        case content
+        when nil
+          raise "No way to install #{name}"
+        when Path
+          Open.read(content) 
+        when String
+          if Path.is_filename?(content) and Open.exists?(content)
+            Open.read(content) 
+          else
+            content
+          end
+        when Hash
+          name = content[:name] || name
+          git = content[:git]
+          src = content[:src]
+          url = content[:url]
+          jar = content[:jar]
+          extra = content[:extra]
+          commands = content[:commands]
+          if git
+            <<-EOF
+
+name='#{name}'
+url='#{git}'
+
+install_git "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif src
+            <<-EOF
+
+name='#{name}'
+url='#{src}'
+
+install_src "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif jar
+            <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          else
+            <<-EOF
+
+name='#{name}'
+url='#{url}'
+
+#{commands}
+            EOF
+          end
+        end
+
+      script = preamble + "\n" + script_text
+      Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
+      CMD.cmd_log('bash', :in => script)
+      Resource.set_software_env(software_dir)
+  end
+
+  def self.set_software_env(software_dir = Path.setup('software'))
+    software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
+      next unless software_dir.exists?
+      Log.medium "Preparing software env at #{software_dir}"
+
+      software_dir = File.expand_path(software_dir)
+      opt_dir = File.join(software_dir, 'opt')
+      bin_dir = File.join(opt_dir, 'bin')
+
+      Misc.env_add 'PATH', bin_dir
+
+      FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
+
+      %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
+        filename = File.join(opt_dir, file)
+        begin
+          FileUtils.touch filename unless File.exist? filename
+        rescue
+          Log.warn("Could not touch #{ filename }")
+        end
+      end
+
+      Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CPLUS_INCLUDE_PATH',dir)
+        Misc.env_add('C_INCLUDE_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.c-paths')
+
+      Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('LIBRARY_PATH',dir)
+        Misc.env_add('LD_LIBRARY_PATH',dir)
+        Misc.env_add('LD_RUN_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.ld-paths')
+
+      Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('PKG_CONFIG_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
+
+      Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
+      end if File.exist? File.join(opt_dir, '.aclocal-paths')
+
+      Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CLASSPATH', "#{dir}")
+      end if File.exist? File.join(opt_dir, '.java-classpaths')
+
+      Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
+        Misc.env_add('CLASSPATH', "#{file}")
+      end
+
+      if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
+        Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
+
+          # Load exports
+          Open.read(file).split("\n").each do |line|
+            next unless line =~ /^\s*export\s+([^=]+)=(.*)/
+            var = $1.strip
+            value = $2.strip
+            value.sub!(/^['"]/,'')
+            value.sub!(/['"]$/,'')
+            value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
+            Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
+            ENV[var] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout/tsv/dumper.rb

@@ -0,0 +1,107 @@
+module TSV
+  class Dumper
+    def self.header_lines(key_field, fields, entry_hash = nil)
+      if Hash === entry_hash 
+        sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
+        preamble = entry_hash[:preamble]
+        header_hash = entry_hash[:header_hash]
+      end
+
+      header_hash = "#" if header_hash.nil?
+
+      preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
+
+      str = "" 
+      str << preamble.strip << "\n" if preamble and not preamble.empty?
+      if fields
+        if fields.empty?
+          str << header_hash << (key_field || "ID").to_s << "\n" 
+        else
+          str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+        end
+      end
+
+      str
+    end
+
+    def self.header(options={})
+      key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options, 
+        :key_field, :fields, :sep, :header_hash, :preamble,
+        :sep => "\t", :header_hash => "#", :preamble => true
+
+      if fields.nil? || key_field.nil?
+        fields_str = nil
+      else
+        fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
+      end
+
+      if preamble && options.values.compact.any?
+        preamble_str = "#: " << IndiferentHash.hash2string(options)
+      else
+        preamble_str = nil
+      end
+
+      [preamble_str, fields_str].compact * "\n"
+    end
+
+
+    attr_accessor :options
+    def initialize(options = {})
+      @sep, @type = IndiferentHash.process_options options, 
+        :sep, :type, 
+        :sep => "\t", :type => :double
+      @options = options
+      @sout, @sin = Open.pipe
+      ConcurrentStream.setup(@sin, :pair => @sout)
+      ConcurrentStream.setup(@sout, :pair => @sin)
+    end
+
+    def init
+      header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
+      @sin.puts header if header and ! header.empty?
+    end
+
+    def add(key, value)
+
+      case @type
+      when :single
+        @sin.puts key + @sep + value
+      when :list, :flat
+        @sin.puts key + @sep + value * @sep
+      when :double
+        @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
+      end
+    end
+
+    def close
+      @sin.close
+      @sin.join
+    end
+
+    def stream
+      @sout
+    end
+
+    def abort(exception=nil)
+      @sin.abort(exception)
+    end
+  end
+
+  def stream
+    iii self.extension_attr_hash
+    dumper = TSV::Dumper.new self.extension_attr_hash
+    dumper.init
+    Thread.new do 
+      Thread.current["name"] = "Dumper thread"
+      self.each do |k,v|
+        dumper.add k, v
+      end
+      dumper.close
+    end
+    dumper.stream
+  end
+
+  def to_s
+    stream.read
+  end
+end

data/lib/scout/tsv/index.rb

@@ -0,0 +1,49 @@
+require_relative 'parser'
+module TSV
+  def self.index(tsv_file, target: 0, order: true, **kwargs)
+    persist, type = IndiferentHash.process_options kwargs,
+      :persist, :persist_type,
+      :persist => false, :persist_type => "HDB"
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+      if filename
+        index = ScoutCabinet.open(filename, true, type)
+        TSV.setup(index, :type => :single)
+        index.extend TSVAdapter 
+      else
+        index = TSV.setup({}, :type => :single)
+      end
+
+      dummy_data = nil
+      if order
+        tmp_index = {}
+        dummy_data = Open.open(tsv_file) do |file|
+          TSV.parse file, key_field: target, type: :double, **kwargs do |k,values|
+            values.each_with_index do |list,i|
+              list.each do |e|
+                tmp_index[e] ||= []
+                tmp_index[e][i] ||= []
+                tmp_index[e][i] << k
+              end
+            end
+          end
+        end
+        tmp_index.each do |e,list|
+          index[e] = list.flatten.compact.uniq.first
+        end
+      else
+        dummy_data = Open.open(tsv_file) do |file|
+          TSV.parse file, key_field: target, type: :flat, **kwargs do |k,values|
+            values.each do |e|
+              index[e] = k unless index.include?(e)
+            end
+          end
+        end
+      end
+      index.key_field = dummy_data.fields * ", "
+      index.fields = [dummy_data.key_field]
+      index
+    end
+  end
+end