scout-gear 5.1.1 → 6.0.0

data/lib/scout/semaphore.rb

@@ -0,0 +1,148 @@
+begin
+  require 'inline'
+  continue = true
+rescue Exception
+  Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
+  continue = false
+end
+
+if continue
+  module ScoutSemaphore
+    inline(:C) do |builder|
+      builder.prefix <<-EOF
+  #include <unistd.h>
+  #include <stdio.h>
+  #include <stdlib.h>
+  #include <semaphore.h>
+  #include <time.h>
+  #include <assert.h>
+  #include <errno.h>
+  #include <signal.h>
+  #include <fcntl.h>
+      EOF
+
+      builder.c_singleton <<-EOF
+  void create_semaphore(char* name, int value){
+    sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+  }
+      EOF
+      builder.c_singleton <<-EOF
+  void delete_semaphore(char* name){
+    sem_unlink(name);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  int wait_semaphore(char* name){
+    int ret;
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    ret = sem_wait(sem);
+    sem_close(sem);
+    return(ret);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  void post_semaphore(char* name){
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    sem_post(sem);
+    sem_close(sem);
+  }
+      EOF
+    end
+
+    SEM_MUTEX = Mutex.new
+    def self.synchronize(sem)
+      ret = ScoutSemaphore.wait_semaphore(sem)
+      raise SemaphoreInterrupted if ret == -1
+      begin
+        yield
+      ensure
+        ScoutSemaphore.post_semaphore(sem)
+      end
+    end
+
+    def self.with_semaphore(size, file = nil)
+      if file.nil?
+        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+      else
+        file = file.gsub('/', '_') if file
+      end
+
+      begin
+        Log.debug "Creating semaphore (#{ size }): #{file}"
+        ScoutSemaphore.create_semaphore(file, size)
+        yield file
+      ensure
+        Log.debug "Removing semaphore #{ file }"
+        ScoutSemaphore.delete_semaphore(file)
+      end
+    end
+
+    def self.fork_each_on_semaphore(elems, size, file = nil)
+
+      TSV.traverse elems, :cpus => size, :bar => "Fork each on semaphore: #{ Misc.fingerprint elems }", :into => Set.new do |elem|
+        elems.annotate elem if elems.respond_to? :annotate
+        begin
+          yield elem
+        rescue Interrupt
+          Log.warn "Process #{Process.pid} was aborted"
+        end
+        nil
+      end
+      nil
+    end
+
+    def self.thread_each_on_semaphore(elems, size)
+      mutex = Mutex.new
+      count = 0
+      cv = ConditionVariable.new
+      wait_mutex = Mutex.new
+
+      begin
+
+        threads = []
+        wait_mutex.synchronize do
+          threads = elems.collect do |elem| 
+            Thread.new(elem) do |elem|
+
+              continue = false
+              mutex.synchronize do
+                while not continue do
+                  if count < size 
+                    continue = true
+                    count += 1
+                  end
+                  mutex.sleep 1 unless continue
+                end
+              end
+
+              begin
+                yield elem
+              rescue Interrupt
+                Log.error "Thread was aborted while processing: #{Misc.fingerprint elem}"
+                raise $!
+              ensure
+                mutex.synchronize do
+                  count -= 1
+                  cv.signal if mutex.locked?
+                end
+              end
+            end
+          end
+        end
+
+        threads.each do |thread| 
+          thread.join 
+        end
+      rescue Exception
+        Log.exception $!
+        Log.info "Ensuring threads are dead: #{threads.length}"
+        threads.each do |thread| thread.kill end
+      end
+    end
+  end 
+end
+

data/lib/scout/simple_opt/doc.rb

@@ -42,10 +42,33 @@ module SOPT
       "=<#{ type }>"
     end
     #extra << " (default: #{Array === default ? (default.length > 3 ? default[0..2]*", " + ', ...' : default*", " ): default})" if default != nil
-    extra << " (default: #{Misc.fingerprint(default)})" if default != nil
+    extra << " (default: #{Log.fingerprint(default)})" if default != nil
     input_str << Log.color(:green, extra)
   end
 
+  def self.input_array_doc(input_array)
+    input_array.collect do |name,type,description,default,options|
+      type = :string if type.nil?
+
+      name = name.to_s
+      shortcut, options = options, nil if String === options || Symbol === options
+
+      case options && options[:shortcut]
+      when FalseClass
+        shortcut = nil
+      when TrueClass, nil
+        shortcut = fix_shortcut(name[0], name)
+      else
+        shortcut = options[:shortcut]
+      end unless shortcut
+
+      shortcut = fix_shortcut(shortcut, name)
+      register(shortcut, name, type, description) unless self.inputs.include? name
+      name  = SOPT.input_format(name, type.to_sym, default, shortcut ) 
+      Misc.format_definition_list_item(name, description)
+    end * "\n"
+  end
+
   def self.input_doc(inputs, input_types = nil, input_descriptions = nil, input_defaults = nil, input_shortcuts = nil)
     type = description = default = nil
     shortcut = ""
@@ -73,10 +96,11 @@ module SOPT
       register(shortcut, name, type, description) unless self.inputs.include? name
 
       name  = SOPT.input_format(name, type.to_sym, default, shortcut) 
-      Misc.format_definition_list_item(name, description, 80, 31, nil)
+      Misc.format_definition_list_item(name, description)
     end * "\n"
   end
 
+
   def self.doc
     doc =<<-EOF
 #{Log.color :magenta}#{command}(1) -- #{summary}

data/lib/scout/work_queue/socket.rb

@@ -0,0 +1,119 @@
+require 'scout/open'
+require 'scout/semaphore'
+require 'scout/exceptions'
+class WorkQueue
+  class Socket
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    def initialize(serializer = nil)
+      @sread, @swrite = Open.pipe
+
+      @serializer = serializer || Marshal
+
+      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @write_sem = @key + '.in'
+      @read_sem = @key + '.out'
+      Log.debug "Creating socket semaphores: #{@key}"
+      ScoutSemaphore.create_semaphore(@write_sem,1)
+      ScoutSemaphore.create_semaphore(@read_sem,1)
+    end
+
+    def clean
+      @cleaned = true
+      @sread.close unless @sread.closed?
+      @swrite.close unless @swrite.closed?
+      Log.low "Destroying socket semaphores: #{[@key] * ", "}"
+      ScoutSemaphore.delete_semaphore(@write_sem)
+      ScoutSemaphore.delete_semaphore(@read_sem)
+    end
+
+
+    def dump(obj)
+      stream = @swrite
+      obj.concurrent_stream = nil if obj.respond_to?(:concurrent_stream)
+      case obj
+      when Integer
+        size_head = [obj,"I"].pack 'La'
+        str = size_head
+      when nil
+        size_head = [0,"N"].pack 'La'
+        str = size_head
+      when String
+        payload = obj
+        size_head = [payload.bytesize,"C"].pack 'La'
+        str = size_head << payload
+      else
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
+      end
+
+      write_length = str.length
+      wrote = stream.write(str) 
+      while wrote < write_length
+        wrote += stream.write(str[wrote..-1]) 
+      end
+    end
+
+    def load
+      stream = @sread
+      size_head = Open.read_stream stream, 5
+
+      size, type = size_head.unpack('La')
+
+      return nil if type == "N"
+      return size.to_i if type == "I"
+      begin
+        payload = Open.read_stream stream, size
+        case type
+        when "S"
+          begin
+            @serializer.load(payload)
+          rescue Exception
+            Log.exception $!
+            raise $!
+          end
+        when "C"
+          payload
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+
+    def closed_read?
+      @sread.closed?
+    end
+
+    def closed_write?
+      @swrite.closed?
+    end
+
+    def close_write
+      self.dump ClosedStream.new
+      @swrite.close unless closed_write?
+    end
+
+    def close_read
+      @sread.close unless closed_read?
+    end
+
+    #{{{ ACCESSOR
+    def push(obj)
+      ScoutSemaphore.synchronize(@write_sem) do
+        self.dump(obj)
+      end
+    end
+
+    def pop
+      ScoutSemaphore.synchronize(@read_sem) do
+        res = self.load
+        raise res if ClosedStream === res
+        res
+      end
+    end
+
+    alias write push
+
+    alias read pop
+  end
+end

data/lib/scout/work_queue/worker.rb

@@ -0,0 +1,54 @@
+class WorkQueue
+  class Worker
+    attr_accessor :pid, :ignore_ouput
+    def initialize
+    end
+
+    def run
+      @pid = Process.fork do
+        yield
+      end
+    end
+
+    def process(input, output, &block)
+      run do
+        begin
+          while obj = input.read
+            if DoneProcessing === obj
+              output.write DoneProcessing.new
+              raise obj 
+            end
+            res = block.call obj
+            output.write res unless ignore_ouput || res == :ignore 
+          end
+        rescue DoneProcessing
+          Log.log "Worker #{Process.pid} done"
+        rescue Exception
+          Log.exception $!
+          exit -1
+        end
+      end
+    end
+
+    def join
+      Log.log "Joining worker #{@pid}"
+      Process.waitpid @pid
+    end
+
+    def exit(status)
+      Log.log "Worker #{@pid} exited with status #{Log.color(:green, status)}"
+    end
+
+    def self.join(workers)
+      workers = [workers] unless Array === workers
+      begin
+        while pid = Process.wait 
+          status = $?
+          worker = workers.select{|w| w.pid == pid }.first
+          worker.exit status.exitstatus if worker
+        end
+      rescue Errno::ECHILD
+      end
+    end
+  end
+end

data/lib/scout/work_queue.rb

@@ -0,0 +1,86 @@
+require_relative 'work_queue/socket'
+require_relative 'work_queue/worker'
+
+class WorkQueue
+  attr_accessor :workers, :worker_proc, :callback
+
+  def initialize(workers = 0, &block)
+    @input = WorkQueue::Socket.new
+    @output = WorkQueue::Socket.new
+    @workers = workers.times.collect{ Worker.new }
+    @worker_proc = block
+    @worker_mutex = Mutex.new
+    @removed_workers = []
+  end
+
+  def add_worker(&block)
+    worker = Worker.new
+    @worker_mutex.synchronize do
+      @workers.push(worker)
+      if block_given?
+        worker.process @input, @output, &block
+      else
+        worker.process @input, @output, &@worker_proc
+      end
+    end
+    worker
+  end
+
+  def ignore_ouput
+    @workers.each{|w| w.ignore_ouput = true }
+  end
+
+  def remove_one_worker
+    @input.write DoneProcessing.new
+  end
+
+  def remove_worker(pid)
+    worker = @worker_mutex.synchronize do
+      Log.debug "Remove #{pid}"
+      @removed_workers.concat(@workers.delete_if{|w| w.pid == pid })
+    end
+  end
+
+  def process(&callback)
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+    @reader = Thread.new do
+      begin
+        while true
+          obj = @output.read
+          if DoneProcessing === obj
+            remove_worker obj.pid if obj.pid
+          else
+            callback.call obj if callback
+          end
+        end
+      rescue Aborted
+      end
+    end if @output
+  end
+
+  def write(obj)
+    @input.write obj
+  end
+
+  def close
+    while @worker_mutex.synchronize{ @workers.length } > 0
+      begin
+        @input.write DoneProcessing.new
+        pid = Process.wait
+        status = $?
+        worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
+        worker.exit $?.exitstatus if worker
+      rescue Errno::ECHILD
+        Thread.pass until @workers.length == 0
+        break
+      end
+    end
+    @reader.raise Aborted if @reader
+  end
+
+  def join
+    @reader.join if @reader
+  end
+end

data/lib/scout/workflow/definition.rb

@@ -33,6 +33,11 @@ module Workflow
     @annotate_next_task[type] << obj
   end
 
+  def annotate_next_task_single(type, obj)
+    @annotate_next_task ||= {}
+    @annotate_next_task[type] = obj
+  end
+
   def dep(*args, &block)
     case args.length
     when 3
@@ -60,13 +65,14 @@ module Workflow
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
-      @tasks[name] = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      @tasks[name] = task
     ensure
       @annotate_next_task = {}
     end
   end
 
   def desc(description)
-    annotate_next_task(:desc, description)
+    annotate_next_task_single(:description, description)
   end
 end

data/lib/scout/workflow/documentation.rb

@@ -1,4 +1,5 @@
 module Workflow
+  attr_accessor :title, :description
 
   def self.doc_parse_first_line(str)
     if str.match(/^([^\n]*)\n\n(.*)/sm)
@@ -45,33 +46,38 @@ module Workflow
     end
   end
 
-  def load_documentation
-    return if @documentation
-    @documentation ||= Workflow.parse_workflow_doc documentation_markdown
-    @documentation[:tasks].each do |task, description|
-      if task.include? "#"
-        workflow, task = task.split("#")
-        workflow = begin
-                     Kernel.const_get workflow
-                   rescue
-                     next
-                   end
-      else
-        workflow = self
-      end
-
-      task = task.to_sym
-      if workflow.tasks.include? task
-        workflow.tasks[task].description = description
-      else
-        Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
-      end
-    end
-  end
-
   attr_accessor :documentation
   def documentation
-    load_documentation if @documentation.nil?
-    @documentation 
+    @documentation ||= begin
+                         documentation = Workflow.parse_workflow_doc documentation_markdown
+
+                         if @description && (documentation[:description].nil? || documentation[:description].empty?)
+                           documentation[:description] = @description 
+                         end
+
+                         if @title && (documentation[:title].nil? || documentation[:title].empty?)
+                           documentation[:title] = @title 
+                         end
+                         documentation[:tasks].each do |task, description|
+                           if task.include? "#"
+                             workflow, task = task.split("#")
+                             workflow = begin
+                                          Kernel.const_get workflow
+                                        rescue
+                                          next
+                                        end
+                           else
+                             workflow = self
+                           end
+
+                           task = task.to_sym
+                           if workflow.tasks.include? task
+                             workflow.tasks[task].description = description
+                           else
+                             Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+                           end
+                         end
+                         documentation
+                       end
   end
 end

data/lib/scout/workflow/step/info.rb

@@ -1,15 +1,21 @@
 class Step
+  SERIALIZER = :json
   def info_file
-    @info_file ||= @path + ".info"
+    @info_file ||= begin
+                     info_file = @path + ".info"
+                     @path.annotate info_file if Path === @path
+                     info_file
+                   end
   end
 
   def load_info
-    @info = Persist.load(info_file, :marshal) || {}
+    @info = Persist.load(info_file, SERIALIZER) || {}
+    IndiferentHash.setup(@info)
     @info_load_time = Time.now
   end
 
-  def save_info
-    Persist.save(@info, info_file, :marshal)
+  def save_info(info = nil)
+    Persist.save(info, info_file, SERIALIZER)
     @info_load_time = Time.now
   end
 
@@ -40,24 +46,18 @@ class Step
         info[key] = value
       end
     end
-    save_info
+    save_info(info)
   end
 
   def set_info(key, value)
     merge_info(key => value)
   end
   
-  def init_info
-    @info = {
-      :status => :waiting
-    }
-  end
-
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path)
     else
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path) + " " + message
     end
   end
 

data/lib/scout/workflow/step/load.rb

@@ -0,0 +1,18 @@
+class Step
+  def self.relocate(path)
+    return path if Open.exists?(path)
+    Path.setup(path) unless Path === path
+    relocated = path.relocate
+    return relocated if Open.exists?(relocated)
+    subpath = path.split("/")[-3..-1] * "/"
+    relocated = Path.setup("var/jobs")[subpath]
+    return relocated if Open.exists?(relocated)
+    path
+  end
+
+  def self.load(path)
+    path = relocate(path) unless Open.exists?(path)
+    raise "Could not load #{path}" unless Open.exists?(path)
+    s = Step.new path
+  end
+end