scout-gear 2.0.0 → 5.1.1

data/lib/scout/resource/produce/rake.rb

@@ -0,0 +1,69 @@
+require_relative '../../misc'
+require_relative '../../path'
+require 'rake'
+
+class Rake::FileTask
+  class << self
+    alias_method :old_define_task, :define_task
+  end
+
+  def self.define_task(file, *args, &block)
+    @@files ||= []
+    @@files << file
+    old_define_task(file, *args, &block)
+  end
+
+  def self.files
+    @@files
+  end
+
+  def self.clear_files
+    @@files = []
+  end
+end
+
+module ScoutRake
+  class TaskNotFound < StandardError; end
+  def self.run(rakefile, dir, task, &block)
+    old_pwd = FileUtils.pwd
+
+    Rake::Task.clear
+    Rake::FileTask.clear_files
+
+    t = nil
+    pid = Process.fork{
+      if block_given?
+        TOPLEVEL_BINDING.receiver.instance_exec &block
+      else
+        if Path.is_filename? rakefile
+          rakefile = rakefile.produce.find
+          load rakefile
+        else
+          TmpFile.with_file(rakefile) do |tmpfile|
+            load tmpfile
+          end
+        end
+      end
+
+      raise TaskNotFound if Rake::Task[task].nil?
+
+      #Misc.pre_fork
+      begin
+        Misc.in_dir(dir) do
+          Rake::Task[task].invoke
+
+          Rake::Task.clear
+          Rake::FileTask.clear_files
+        end
+      rescue Exception
+        Log.error "Error in rake: #{$!.message}"
+        Log.exception $!
+        Kernel.exit! -1
+      end
+      Kernel.exit! 0
+    }
+    Process.waitpid(pid)
+    raise "Rake failed" unless $?.success?
+
+  end
+end

data/lib/scout/resource/produce.rb

@@ -0,0 +1,246 @@
+require_relative '../open'
+require_relative '../tmpfile'
+require_relative 'produce/rake'
+
+module Resource
+  def claim(path, type, content = nil, &block)
+    if type == :rake
+      @rake_dirs ||= {}
+      @rake_dirs[path] = content || block
+    else
+      @resources ||= {}
+      @resources[path] = [type, content || block]
+    end
+  end
+
+  def rake_for(path)
+    @rake_dirs ||= {}
+    @rake_dirs.select{|dir, content|
+      Misc.path_relative_to(dir, path)
+    }.sort_by{|dir, content|
+      dir.length
+    }.last
+  end
+
+  def has_rake(path)
+    !! rake_for(path)
+  end
+
+  def run_rake(path, rakefile, rake_dir)
+    task = Misc.path_relative_to rake_dir, path
+    rakefile = rakefile.produce if rakefile.respond_to? :produce
+    rakefile = rakefile.find if rakefile.respond_to? :find
+
+    rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
+
+    begin
+      if Proc === rakefile
+        ScoutRake.run(nil, rake_dir, task, &rakefile)
+      else
+        ScoutRake.run(rakefile, rake_dir, task)
+      end
+    rescue Rake::TaskNotFound
+      if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
+        raise $! 
+      end
+      task = File.join(File.basename(rake_dir), task)
+      rake_dir = File.dirname(rake_dir)
+      retry
+    end
+  end
+
+  def produce(path, force = false)
+    case
+    when @resources.include?(path)
+      type, content = @resources[path]
+    when (Path === path && @resources.include?(path.original))
+      type, content = @resources[path.original]
+    when has_rake(path)
+      type = :rake
+      rake_dir, content = rake_for(path)
+      rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
+    else
+      if path !~ /\.(gz|bgz)$/
+        begin
+          produce(path.annotate(path + '.gz'), force)
+        rescue ResourceNotFound
+          begin
+            produce(path.annotate(path + '.bgz'), force)
+          rescue ResourceNotFound
+            raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+          end
+        end
+      else
+        raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+      end
+    end
+
+    if path.respond_to?(:find) 
+      final_path = force ? path.find(:default) : path.find
+    else
+      final_path = path
+    end
+
+    if type and not File.exist?(final_path) or force
+      Log.medium "Producing: (#{self.to_s}) #{ final_path }"
+      lock_filename = TmpFile.tmp_for_file(final_path, :dir => lock_dir)
+
+      Open.lock lock_filename do
+        FileUtils.rm_rf final_path if force and File.exist? final_path
+
+        if ! File.exist?(final_path) || force 
+
+          begin
+            case type
+            when :string
+              Open.sensible_write(final_path, content)
+            when :csv
+              raise "TSV/CSV Not implemented yet"
+              #require 'rbbt/tsv/csv'
+              #tsv = TSV.csv Open.open(content)
+              #Open.sensible_write(final_path, tsv.to_s)
+            when :url
+              options = {}
+              options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)
+              Open.sensible_write(final_path, Open.open(content, options))
+            when :proc
+              data = case content.arity
+                     when 0
+                       content.call
+                     when 1
+                       content.call final_path
+                     end
+              case data
+              when String, IO, StringIO
+                Open.sensible_write(final_path, data) 
+              when Array
+                Open.sensible_write(final_path, data * "\n")
+              when TSV
+                Open.sensible_write(final_path, data.dumper_stream) 
+              when TSV::Dumper
+                Open.sensible_write(final_path, data.stream) 
+              when nil
+              else
+                raise "Unkown object produced: #{Log.fingerprint data}"
+              end
+            when :rake
+              run_rake(path, content, rake_dir)
+            when :install
+              Log.debug "Installing software: #{path}"
+
+              $set_software_env = false unless File.exist? path
+              
+              software_dir = path.resource.root.software.find :user
+              helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
+              #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)
+
+              preamble = <<-EOF
+#!/bin/bash
+
+RBBT_SOFTWARE_DIR="#{software_dir}"
+
+INSTALL_HELPER_FILE="#{helper_file}"
+source "$INSTALL_HELPER_FILE"
+              EOF
+
+              content = content.call if Proc === content
+
+              content = if content =~ /git:|\.git$/
+                          {:git => content}
+                        else
+                          {:src => content}
+                        end if String === content and Open.remote?(content)
+
+              script_text = case content
+                            when nil
+                              raise "No way to install #{path}"
+                            when Path
+                              Open.read(content) 
+                            when String
+                              if Path.is_filename?(content) and Open.exists?(content)
+                                Open.read(content) 
+                              else
+                                content
+                              end
+                            when Hash
+                              name = content[:name] || File.basename(path)
+                              git = content[:git]
+                              src = content[:src]
+                              url = content[:url]
+                              jar = content[:jar]
+                              extra = content[:extra]
+                              commands = content[:commands]
+                              if git
+                                <<-EOF
+
+name='#{name}'
+url='#{git}'
+
+install_git "$name" "$url" #{extra}
+
+#{commands}
+                                EOF
+                              elsif src
+                                <<-EOF
+
+name='#{name}'
+url='#{src}'
+
+install_src "$name" "$url" #{extra}
+
+#{commands}
+                                EOF
+                              elsif jar
+                                <<-EOF
+
+name='#{name}'
+url='#{jar}'
+
+install_jar "$name" "$url" #{extra}
+
+#{commands}
+                                EOF
+                              else
+                                <<-EOF
+
+name='#{name}'
+url='#{url}'
+
+#{commands}
+                                EOF
+                              end
+                            end
+
+              script = preamble + "\n" + script_text
+              Log.debug "Installing software with script:\n" << script
+              CMD.cmd_log('bash', :in => script)
+
+              set_software_env(software_dir) unless $set_software_env
+              $set_software_env = true
+            else
+              raise "Could not produce #{ resource }. (#{ type }, #{ content })"
+            end
+          rescue
+            FileUtils.rm_rf final_path if File.exist? final_path
+            raise $!
+          end
+        end
+      end
+    end
+
+    # After producing a file, make sure we recheck all locations, the file
+    # might have appeared with '.gz' extension for instance
+    path.instance_variable_set("@path", {})
+
+    path
+  end
+
+end
+
+module Path
+  def produce(force = false)
+    return self if ! force && Open.exist?(self)
+    self.pkgdir.produce self if Resource === self.pkgdir
+    return self
+  end
+end

data/lib/scout/resource/scout.rb

@@ -0,0 +1,3 @@
+module Scout
+  extend Resource
+end

data/lib/scout/resource.rb

@@ -0,0 +1,37 @@
+require_relative 'log'
+require_relative 'path'
+require_relative 'resource/produce'
+
+module Resource
+  extend MetaExtension
+  extension_attr :pkgdir, :libdir, :subdir, :resources, :rake_dirs, :path_maps, :lock_dir
+
+  def self.default_lock_dir
+    Path.setup('tmp/produce_locks').find
+  end
+
+  def subdir
+    @subdir ||= ""
+  end
+
+  def lock_dir
+    @lock_dir ||= Resource.default_lock_dir
+  end
+
+  def pkgdir
+    @pkgdir ||= Path.default_pkgdir
+  end
+
+  def root
+    Path.setup(subdir, self, self.libdir, @path_maps)
+  end
+
+  def method_missing(name, prev = nil, *args)
+    if prev.nil?
+      root.send(name, *args)
+    else
+      root.send(name, prev, *args)
+    end
+  end
+end
+

data/lib/scout/simple_opt/accessor.rb

@@ -1,6 +1,6 @@
 module SOPT
   class << self
-    attr_accessor :inputs, :input_shortcuts, :input_types, :input_descriptions, :input_defaults
+    attr_writer :inputs, :input_shortcuts, :input_types, :input_descriptions, :input_defaults
   end
 
   def self.all

data/lib/scout/simple_opt/doc.rb

@@ -2,7 +2,7 @@ require_relative '../log'
 module SOPT
 
   class << self
-    attr_accessor :command, :summary, :synopsys, :description
+    attr_writer :command, :summary, :synopsys, :description
   end
 
   def self.command
@@ -73,32 +73,12 @@ module SOPT
       register(shortcut, name, type, description) unless self.inputs.include? name
 
       name  = SOPT.input_format(name, type.to_sym, default, shortcut) 
-      description 
       Misc.format_definition_list_item(name, description, 80, 31, nil)
     end * "\n"
   end
 
   def self.doc
-    doc = <<-EOF
-#{Log.color :magenta}#{command}(1) -- #{summary}
-#{"=" * (command.length + summary.length + 7)}#{Log.color :reset}
-
-#{ Log.color :magenta, "## SYNOPSYS"}
-
-#{Log.color :blue, synopsys}
-
-#{ Log.color :magenta, "## DESCRIPTION"}
-
-#{Misc.format_paragraph description}
-
-#{ Log.color :magenta, "## OPTIONS"}
-
-#{input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)}
-    EOF
-  end
-
-  def self.doc
-    doc = <<-EOF
+    doc =<<-EOF
 #{Log.color :magenta}#{command}(1) -- #{summary}
 #{"=" * (command.length + summary.length + 7)}#{Log.color :reset}
 
@@ -116,5 +96,7 @@ module SOPT
 
     doc << Log.color(:magenta, "## OPTIONS") << "\n\n"
     doc << input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)
+
+    doc
   end
 end

data/lib/scout/simple_opt/parse.rb

@@ -44,23 +44,24 @@ module SOPT
   end
 
   def self.parse(opt_str)
-    info = {}
-
     inputs = []
+
     if opt_str.include? "\n"
       re = /\n+/
     else
       re = /:/
     end
+
     opt_str.split(re).each do |entry|
       entry.strip!
       next if entry.empty?
-      names, _sep, description = entry.partition /\s+/
+      names, _sep, description = entry.partition(/\s+/)
       short, long, asterisk = names.match(/\s*(?:-(.+))?(?:--(.+?))([*])?$/).values_at 1,2,3 
 
       inputs << long
       register short, long, asterisk, description
     end
+
     inputs
   end
 end

data/lib/scout/tmpfile.rb

@@ -1,7 +1,10 @@
-require 'fileutils'
 require_relative 'misc'
+require_relative 'log'
+require 'fileutils'
 
 module TmpFile
+  MAX_FILE_LENGTH = 150
+
   def self.user_tmp(subdir = nil)
     if subdir
       File.join(ENV["HOME"],"/tmp/scout", subdir)
@@ -89,4 +92,39 @@ module TmpFile
       end
     end
   end
+
+  def self.tmp_for_file(file, tmp_options = {}, other_options = {})
+    tmp_for_file = IndiferentHash.process_options tmp_options, :file
+    return tmp_for_file unless tmp_for_file.nil?
+
+    prefix = IndiferentHash.process_options tmp_options, :prefix
+
+    if prefix.nil?
+      perfile = file.to_s.gsub(/\//, '>') 
+    else
+      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
+    end
+
+    perfile.sub!(/\.b?gz$/,'')
+
+    if other_options.include? :filters
+      other_options[:filters].each do |match,value|
+        perfile = perfile + "&F[#{match}=#{Misc.digest(value)}]"
+      end
+    end
+
+    persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir 
+    Path.setup(persistence_dir) unless Path === persistence_dir
+
+    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    clean_options = other_options.dup
+    clean_options.delete :unnamed
+    clean_options.delete "unnamed"
+
+    filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
+
+    filename  += ":" << Misc.digest(clean_options) unless clean_options.empty?
+
+    persistence_dir[filename]
+  end
 end

data/lib/scout/workflow/definition.rb

@@ -0,0 +1,72 @@
+require_relative '../meta_extension'
+
+module Workflow
+  extend MetaExtension
+  extension_attr :name, :tasks
+
+  class << self
+    attr_accessor :directory
+
+    def directory
+      @directory ||= Path.setup('var/jobs')
+    end
+
+  end
+
+  def name
+    @name ||= self.to_s
+  end
+
+  attr_accessor :directory
+  def directory
+    @directory ||= Workflow.directory[name]
+  end
+
+  def directory=(directory)
+    @directory = directory
+    @tasks.each{|name,d| d.directory = directory[name] } if @tasks
+  end
+
+  def annotate_next_task(type, obj)
+    @annotate_next_task ||= {}
+    @annotate_next_task[type] ||= []
+    @annotate_next_task[type] << obj
+  end
+
+  def dep(*args, &block)
+    case args.length
+    when 3
+      workflow, task, options = args
+    when 2
+      if Hash === args.last
+        task, options = args
+      else
+        workflow, task = args
+      end
+    when 1
+      task = args.first
+    end
+    workflow = self if workflow.nil?
+    options = {} if options.nil?
+    annotate_next_task :deps, [workflow, task, options, block, args]
+  end
+
+  def input(*args)
+    annotate_next_task(:inputs, args)
+  end
+
+  def task(name_and_type, &block)
+    name, type = name_and_type.collect.first
+    @tasks ||= IndiferentHash.setup({})
+    begin
+      @annotate_next_task ||= {}
+      @tasks[name] = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+    ensure
+      @annotate_next_task = {}
+    end
+  end
+
+  def desc(description)
+    annotate_next_task(:desc, description)
+  end
+end

data/lib/scout/workflow/documentation.rb

@@ -0,0 +1,77 @@
+module Workflow
+
+  def self.doc_parse_first_line(str)
+    if str.match(/^([^\n]*)\n\n(.*)/sm)
+      str.replace $2
+      $1
+    else
+      ""
+    end
+  end
+
+  def self.doc_parse_up_to(str, pattern, keep = false)
+    pre, _pat, _post = str.partition pattern
+    if _pat
+      [pre, (keep ? _pat << _post : _post)]
+    else
+      _post
+    end
+  end
+
+  def self.doc_parse_chunks(str, pattern)
+    parts = str.split(pattern)
+    return {} if parts.length < 2
+    tasks = Hash[*parts[1..-1].collect{|v| v.strip}]
+    tasks.delete_if{|t,d| d.empty?}
+    tasks
+  end
+
+  def self.parse_workflow_doc(doc)
+    title = doc_parse_first_line doc
+    description, task_info = doc_parse_up_to doc, /^# Tasks/i
+    task_description, tasks = doc_parse_up_to task_info, /^##/, true
+    tasks = doc_parse_chunks tasks, /## (.*)/ 
+    {:title => title.strip, :description => description.strip, :task_description => task_description.strip, :tasks => tasks}
+  end
+
+  def documentation_markdown
+    return "" if @libdir.nil?
+    file = @libdir['workflow.md'].find
+    file = @libdir['README.md'].find unless file.exists?
+    if file.exists?
+      file.read
+    else
+      ""
+    end
+  end
+
+  def load_documentation
+    return if @documentation
+    @documentation ||= Workflow.parse_workflow_doc documentation_markdown
+    @documentation[:tasks].each do |task, description|
+      if task.include? "#"
+        workflow, task = task.split("#")
+        workflow = begin
+                     Kernel.const_get workflow
+                   rescue
+                     next
+                   end
+      else
+        workflow = self
+      end
+
+      task = task.to_sym
+      if workflow.tasks.include? task
+        workflow.tasks[task].description = description
+      else
+        Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+      end
+    end
+  end
+
+  attr_accessor :documentation
+  def documentation
+    load_documentation if @documentation.nil?
+    @documentation 
+  end
+end