scout-essentials 1.0.0

data/lib/scout/resource/produce.rb

@@ -0,0 +1,151 @@
+require_relative '../open'
+require_relative '../tmpfile'
+require_relative 'produce/rake'
+
+module Resource
+  def claim(path, type, content = nil, &block)
+    if type == :rake
+      @rake_dirs ||= {}
+      @rake_dirs[path] = content || block
+    else
+      @resources ||= {}
+      @resources[path] = [type, content || block]
+    end
+  end
+
+  def rake_for(path)
+    @rake_dirs ||= {}
+    @rake_dirs.select{|dir, content|
+      Misc.path_relative_to(dir, path)
+    }.sort_by{|dir, content|
+      dir.length
+    }.last
+  end
+
+  def has_rake(path)
+    !! rake_for(path)
+  end
+
+  def run_rake(path, rakefile, rake_dir)
+    task = Misc.path_relative_to rake_dir, path
+    rakefile = rakefile.produce if rakefile.respond_to? :produce
+    rakefile = rakefile.find if rakefile.respond_to? :find
+
+    rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
+
+    begin
+      if Proc === rakefile
+        ScoutRake.run(nil, rake_dir, task, &rakefile)
+      else
+        ScoutRake.run(rakefile, rake_dir, task)
+      end
+    rescue ScoutRake::TaskNotFound
+      if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
+        raise $! 
+      end
+      task = File.join(File.basename(rake_dir), task)
+      rake_dir = File.dirname(rake_dir)
+      retry
+    end
+  end
+
+  def produce(path, force = false)
+    case
+    when (@resources && @resources.include?(path))
+      type, content = @resources[path]
+    when (Path === path && @resources && @resources.include?(path.original))
+      type, content = @resources[path.original]
+    when has_rake(path)
+      type = :rake
+      rake_dir, content = rake_for(path)
+      rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
+    else
+      if path !~ /\.(gz|bgz)$/
+        begin
+          produce(path.annotate(path + '.gz'), force)
+        rescue ResourceNotFound
+          begin
+            produce(path.annotate(path + '.bgz'), force)
+          rescue ResourceNotFound
+            raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+          end
+        end
+      else
+        raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+      end
+    end
+
+    if path.respond_to?(:find) 
+      final_path = force ? path.find(:default) : path.find
+    else
+      final_path = path
+    end
+
+    if type and not File.exist?(final_path) or force
+      Log.medium "Producing: (#{self.to_s}) #{ final_path }"
+      lock_filename = TmpFile.tmp_for_file(final_path, :dir => lock_dir)
+
+      Open.lock lock_filename do
+        FileUtils.rm_rf final_path if force and File.exist? final_path
+
+        if ! File.exist?(final_path) || force 
+
+          begin
+            case type
+            when :string
+              Open.sensible_write(final_path, content)
+            when :csv
+              raise "TSV/CSV Not implemented yet"
+              #require 'rbbt/tsv/csv'
+              #tsv = TSV.csv Open.open(content)
+              #Open.sensible_write(final_path, tsv.to_s)
+            when :url
+              options = {}
+              options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)
+              Open.sensible_write(final_path, Open.open(content, options))
+            when :proc
+              data = case content.arity
+                     when 0
+                       content.call
+                     when 1
+                       content.call final_path
+                     end
+              case data
+              when String, IO, StringIO
+                Open.sensible_write(final_path, data) 
+              when Array
+                Open.sensible_write(final_path, data * "\n")
+              when TSV
+                Open.sensible_write(final_path, data.dumper_stream) 
+              when TSV::Dumper
+                Open.sensible_write(final_path, data.stream) 
+              when nil
+              else
+                raise "Unkown object produced: #{Log.fingerprint data}"
+              end
+            when :rake
+              run_rake(path, content, rake_dir)
+            when :install
+              software_dir = self.root.software
+              name = File.basename(path)
+              Resource.install(content, name, software_dir)
+              set_software_env(software_dir)
+            else
+              raise "Could not produce #{ resource }. (#{ type }, #{ content })"
+            end
+          rescue
+            FileUtils.rm_rf final_path if File.exist? final_path
+            raise $!
+          end
+        end
+      end
+    end
+
+    # After producing a file, make sure we recheck all locations, the file
+    # might have appeared with '.gz' extension for instance
+    path.instance_variable_set("@path", {})
+
+    path
+  end
+
+end

data/lib/scout/resource/scout.rb

@@ -0,0 +1,3 @@
+module Scout
+  extend Resource
+end

data/lib/scout/resource/software.rb

@@ -0,0 +1,178 @@
+module Resource
+
+  def self.install_helpers
+    File.expand_path(Scout.share.software.install_helpers.find(:lib))
+  end
+
+  def self.install(content, name, software_dir = Path.setup('software'), &block)
+    software_dir ||= Path.setup('software')
+    software_dir = software_dir.find(:user) if Path === software_dir
+
+    content = block if block_given?
+
+    preamble = <<-EOF
+#!/bin/bash
+
+SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{install_helpers}"
+source "$INSTALL_HELPER_FILE"
+    EOF
+
+    content = content.call if Proc === content
+
+    name = content[:name] if Hash === content && content.include?(:name)
+    content = 
+      if content =~ /git:|\.git$/
+        {:git => content}
+      else
+        {:src => content}
+      end if String === content and Open.remote?(content)
+
+      script_text = 
+        case content
+        when nil
+          raise "No way to install #{name}"
+        when Path
+          Open.read(content) 
+        when String
+          if Path.is_filename?(content) and Open.exists?(content)
+            Open.read(content) 
+          else
+            content
+          end
+        when Hash
+          name = content[:name] || name
+          git = content[:git]
+          src = content[:src]
+          url = content[:url]
+          jar = content[:jar]
+          extra = content[:extra]
+          commands = content[:commands]
+          if git
+            <<-EOF
+
+name='#{name}'
+url='#{git}'
+
+install_git "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif src
+            <<-EOF
+
+name='#{name}'
+url='#{src}'
+
+install_src "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          elsif jar
+            <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
+#{commands}
+            EOF
+          else
+            <<-EOF
+
+name='#{name}'
+url='#{url}'
+
+#{commands}
+            EOF
+          end
+        end
+
+      script = preamble + "\n" + script_text
+      Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
+      CMD.cmd_log('bash', :in => script)
+      Resource.set_software_env(software_dir)
+  end
+
+  def self.set_software_env(software_dir = Path.setup('software'))
+    software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
+      next unless software_dir.exists?
+      Log.medium "Preparing software env at #{software_dir}"
+
+      software_dir = File.expand_path(software_dir)
+      opt_dir = File.join(software_dir, 'opt')
+      bin_dir = File.join(opt_dir, 'bin')
+
+      Misc.env_add 'PATH', bin_dir
+
+      FileUtils.mkdir_p opt_dir unless File.exist? opt_dir
+
+      %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
+        filename = File.join(opt_dir, file)
+        begin
+          FileUtils.touch filename unless File.exist? filename
+        rescue
+          Log.warn("Could not touch #{ filename }")
+        end
+      end
+
+      Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CPLUS_INCLUDE_PATH',dir)
+        Misc.env_add('C_INCLUDE_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.c-paths')
+
+      Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('LIBRARY_PATH',dir)
+        Misc.env_add('LD_LIBRARY_PATH',dir)
+        Misc.env_add('LD_RUN_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.ld-paths')
+
+      Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('PKG_CONFIG_PATH',dir)
+      end if File.exist? File.join(opt_dir, '.pkgconfig-paths')
+
+      Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
+      end if File.exist? File.join(opt_dir, '.aclocal-paths')
+
+      Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
+        dir = line.chomp
+        dir = File.join(opt_dir, dir) unless dir[0] == "/"
+        Misc.env_add('CLASSPATH', "#{dir}")
+      end if File.exist? File.join(opt_dir, '.java-classpaths')
+
+      Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
+        Misc.env_add('CLASSPATH', "#{file}")
+      end
+
+      if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
+        Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|
+
+          # Load exports
+          Open.read(file).split("\n").each do |line|
+            next unless line =~ /^\s*export\s+([^=]+)=(.*)/
+            var = $1.strip
+            value = $2.strip
+            value.sub!(/^['"]/,'')
+            value.sub!(/['"]$/,'')
+            value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
+            Log.debug "Set variable export from .post_install: #{Log.fingerprint [var,value]*"="}"
+            ENV[var] = value
+          end
+        end
+      end
+    end
+  end
+
+  self.set_software_env
+end

data/lib/scout/resource/util.rb

@@ -0,0 +1,59 @@
+module Resource
+  def identify(path)
+    return path unless path.start_with?("/")
+    path_maps = path.path_maps if Path === path
+    path_maps ||= self.path_maps || Path.path_maps
+    path = File.expand_path(path)
+    path += "/" if File.directory?(path)
+
+    map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
+    map_order -= [:current, "current"]
+
+    choices = []
+    map_order.uniq.each do |name|
+      pattern = path_maps[name]
+      pattern = path_maps[pattern] while Symbol === pattern
+      next if pattern.nil?
+
+      pattern = pattern.sub('{PWD}', Dir.pwd)
+      pattern = pattern.sub('{HOME}', ENV["HOME"])
+      if String ===  pattern and pattern.include?('{')
+        regexp = "^" + pattern
+          .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
+          .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
+        "(?:/(?<REST>.*))?/?$"
+        if m = path.match(regexp) 
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
+            unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
+              m.named_captures.include?(c) ? m[c] : nil
+            }.compact * "/"
+            unlocated.gsub!(/\/+/,'/')
+            if self.subdir && ! self.subdir.empty?
+              subdir = self.subdir
+              subdir += "/" unless subdir.end_with?("/")
+              unlocated[subdir] = "" 
+            end
+            choices << self.annotate(unlocated)
+          end
+        end
+      end
+    end
+
+    identified = choices.sort_by{|s| s.length }.first
+
+    Path.setup(identified || path, self, nil, path_maps)
+  end
+
+  def self.identify(path)
+    resource = path.pkgdir if Path === path
+    resource = Scout unless Resource === resource
+    unlocated = resource.identify path
+  end
+
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    unlocated = identify(path)
+    unlocated.find
+  end
+end
+

data/lib/scout/resource.rb

@@ -0,0 +1,40 @@
+require_relative 'log'
+require_relative 'path'
+require_relative 'resource/produce'
+require_relative 'resource/path'
+require_relative 'resource/open'
+require_relative 'resource/util'
+require_relative 'resource/software'
+
+module Resource
+  extend MetaExtension
+  extension_attr :pkgdir, :libdir, :subdir, :resources, :rake_dirs, :path_maps, :lock_dir
+
+  def self.default_lock_dir
+    Path.setup('tmp/produce_locks').find
+  end
+
+  def subdir
+    @subdir ||= ""
+  end
+
+  def lock_dir
+    @lock_dir ||= Resource.default_lock_dir
+  end
+
+  def pkgdir
+    @pkgdir ||= Path.default_pkgdir
+  end
+
+  def root
+    Path.setup(subdir, self, self.libdir, @path_maps)
+  end
+
+  def method_missing(name, prev = nil, *args)
+    if prev.nil?
+      root.send(name, *args)
+    else
+      root.send(name, prev, *args)
+    end
+  end
+end

data/lib/scout/simple_opt/accessor.rb

@@ -0,0 +1,54 @@
+module SOPT
+  class << self
+    attr_writer :inputs, :input_shortcuts, :input_types, :input_descriptions, :input_defaults
+  end
+
+  def self.all
+    @all ||= {}
+  end
+
+  def self.shortcuts
+    @shortcuts ||= {}
+  end
+  
+  def self.inputs 
+    @inputs ||= []
+  end
+
+  def self.input_shortcuts 
+    @input_shortcuts ||= {}
+  end
+
+  def self.input_types 
+    @input_types ||= {}
+  end
+
+  def self.input_descriptions 
+    @input_descriptions ||= {}
+  end
+
+  def self.input_defaults 
+    @input_defaults ||= {}
+  end
+
+  def self.reset
+    @shortcuts = {}
+    @all = {}
+  end
+
+  def self.delete_inputs(inputs)
+    inputs.each do |input|
+      input = input.to_s
+      self.shortcuts.delete self.input_shortcuts.delete(input)
+      self.inputs.delete input
+      self.input_types.delete input
+      self.input_defaults.delete input
+      self.input_descriptions.delete input
+    end
+  end
+
+  def self.usage
+    puts SOPT.doc
+    exit 0
+  end
+end

data/lib/scout/simple_opt/doc.rb

@@ -0,0 +1,126 @@
+require_relative '../log'
+module SOPT
+
+  class << self
+    attr_writer :command, :summary, :synopsys, :description
+  end
+
+  def self.command
+    @command ||= File.basename($0)
+  end
+
+  def self.summary
+    @summary ||= ""
+  end
+
+  def self.synopsys
+    @synopsys ||= begin
+                    "#{command} " <<
+                    inputs.collect{|name|
+                      "[" << input_format(name, input_types[name] || :string, input_defaults[name], input_shortcuts[name]).sub(/:$/,'') << "]"
+                    } * " "
+                  end
+  end
+
+  def self.description
+    @description ||= "Missing"
+  end
+
+  def self.input_format(name, type = nil, default = nil, short = nil)
+    input_str = (short.nil? or short.empty?) ? "--#{name}" : "-#{short},--#{name}"
+    input_str = Log.color(:blue, input_str)
+    extra = case type
+    when nil
+      ""
+    when :boolean
+      "[=false]" 
+    when :tsv, :text
+      "=<file|->"
+    when :array
+      "=<list|file|->"
+    else
+      "=<#{ type }>"
+    end
+    #extra << " (default: #{Array === default ? (default.length > 3 ? default[0..2]*", " + ', ...' : default*", " ): default})" if default != nil
+    extra << " (default: #{Log.fingerprint(default)})" if default != nil
+    input_str << Log.color(:green, extra)
+  end
+
+  def self.input_array_doc(input_array)
+    input_array.collect do |name,type,description,default,options|
+      type = :string if type.nil?
+
+      name = name.to_s
+      shortcut, options = options, nil if String === options || Symbol === options
+
+      case options && options[:shortcut]
+      when FalseClass
+        shortcut = nil
+      when TrueClass, nil
+        shortcut = fix_shortcut(name[0], name)
+      else
+        shortcut = options[:shortcut]
+      end unless shortcut
+
+      shortcut = fix_shortcut(shortcut, name)
+      register(shortcut, name, type, description) unless self.inputs.include? name
+      name  = SOPT.input_format(name, type.to_sym, default, shortcut ) 
+      Misc.format_definition_list_item(name, description)
+    end * "\n"
+  end
+
+  def self.input_doc(inputs, input_types = nil, input_descriptions = nil, input_defaults = nil, input_shortcuts = nil)
+    type = description = default = nil
+    shortcut = ""
+    seen = []
+    inputs.collect do |name|
+      next if seen.include? name
+      seen << name
+
+      type = input_types[name] unless input_types.nil?
+      description = input_descriptions[name] unless input_descriptions.nil?
+      default = input_defaults[name] unless input_defaults.nil?
+
+      name = name.to_s
+
+      case input_shortcuts
+      when nil, FalseClass
+        shortcut = nil
+      when Hash
+        shortcut = input_shortcuts[name] 
+      when TrueClass
+        shortcut = fix_shortcut(name[0], name)
+      end
+
+      type = :string if type.nil?
+      register(shortcut, name, type, description) unless self.inputs.include? name
+
+      name  = SOPT.input_format(name, type.to_sym, default, shortcut) 
+      Misc.format_definition_list_item(name, description)
+    end * "\n"
+  end
+
+
+  def self.doc
+    doc =<<-EOF
+#{Log.color :magenta}#{command}(1) -- #{summary}
+#{"=" * (command.length + summary.length + 7)}#{Log.color :reset}
+
+    EOF
+
+    if synopsys and not synopsys.empty?
+      doc << Log.color(:magenta, "## SYNOPSYS") << "\n\n"
+      doc << Log.color(:blue, synopsys) << "\n\n"
+    end
+
+    if description and not description.empty?
+      doc << Log.color(:magenta, "## DESCRIPTION") << "\n\n"
+      doc << Misc.format_paragraph(description) << "\n\n"
+    end
+
+    doc << Log.color(:magenta, "## OPTIONS") << "\n\n"
+    doc << input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)
+
+    doc
+  end
+end

data/lib/scout/simple_opt/get.rb

@@ -0,0 +1,57 @@
+module SOPT
+  GOT_OPTIONS= IndiferentHash.setup({})
+  def self.current_options=(options)
+    @@current_options = options
+  end
+  def self.consume(args = ARGV)
+    i = 0
+    @@current_options ||= {}
+    while i < args.length do
+      current = args[i]
+      break if current == "--"
+      if m = current.match(/--?(.+?)(?:=(.+))?$/)
+        key = $1
+        value = $2
+
+        input = inputs.include?(key)? key : shortcuts[key]
+
+        if input.nil?
+          i += 1
+          next
+        else
+          args.delete_at i
+        end
+      else
+        i += 1
+        next
+      end
+
+      if input_types[input] == :string
+        value = args.delete_at(i) if value.nil?
+        @@current_options[input] = value
+      else
+        if value.nil? and %w(F false FALSE no).include?(args[i])
+          Log.warn "Boolean values are best specified as #{current}=[true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value" 
+          value = args.delete_at(i)
+        end
+        @@current_options[input] = %w(F false FALSE no).include?(value)? false : true
+      end
+    end
+
+    IndiferentHash.setup @@current_options
+    GOT_OPTIONS.merge! @@current_options
+
+    @@current_options
+  end
+  
+  def self.get(opt_str)
+    SOPT.parse(opt_str)
+    SOPT.consume(ARGV)
+  end
+
+  def self.require(options, *parameters)
+    parameters.flatten.each do |parameter|
+      raise ParameterException, "Parameter '#{ Log.color :blue, parameter }' not given" if options[parameter].nil?
+    end
+  end
+end