jobserver 0.1.4

data/lib/jobserver.rb

@@ -0,0 +1,619 @@
+#
+# Jobserver version 0.1.4
+#
+# Copyright (c) 2004 Christian Bang <cbang@web.de>
+#
+# This program is free software.
+# You can distribute/modify this program under the same terms of ruby.
+#
+# The class +JobServer+ supplies capabilities to execute jobs on the
+# local- or on remote hosts.
+# * Jobs encapsulate the call of a command in a shell on a (remote) host.
+# * Each client is controlled by a _worker_-thread on the server.
+#   When a client is idle it receives the next job from the queue.
+# * Remote jobs will be launched using ssh. Therefore you must
+#   configure your ssh keys without authentification password.
+# * A common home directory is _not_ needed for the clients.
+#   Different machine architectures as well as binaries are possible.
+# * Data for the clients can be saved to files on the client machines.
+#   See store_data.
+# == How to use the jobserver
+# 1. Set up a default command to be launched in the client's working
+#    directory.
+# 2. Write a client output handler that parses the output lines,
+#    retrieves the results and stores them for you.
+# 3. Create your jobs, give them the parameter settings they need
+#    to run the client command. Give them additional data if you want.
+# 4. Create a JobServer instance.
+# 5. Declare the client machines you want to use.
+# 6. Run the JobServer and wait for all jobs to finish.
+# See an example in the description of +JobServer+ or have a look at
+# the examples/ directory of the jobserver package.
+
+require 'thread'
+require 'monitor'
+require 'tempfile'
+
+###### Job ####################################################################
+
+# Use +Job+ objects to generate a _jobQueue_ array for the +JobServer+.
+# See +JobServer+ for an example.
+class Job
+  WAITING = 0
+  RUNNING = 1
+  FAILED = 2
+  SUCCESS = 3
+  @@default_client_command = ""
+  @@nicelevel = 19  # default process priority for the command
+  @@jobNumber = 0   # class variable to give each new job a new number
+  # Returns the default client command that will be used if none is specified in #new.
+  def Job::default_client_command() @@default_client_command end
+  # Sets the default client command that will be used if none is specified in #new.
+  def Job::default_client_command=(s) @@default_client_command = s end
+  # Returns the nice level (priority) used when the client is run. Default is 19.
+  # For further information about +nice+ see the man-pages for +nice+.
+  def Job::nicelevel() @@nicelevel end
+  # Sets the nice level (priority) used when the client is run. Default is 19.
+  # If you set it to nil, the +nice+ command will not be used.
+  def Job::nicelevel=(v) @@nicelevel = v end
+  @@verbose = 1
+  # Sets the verbose level. 0 means no output, > 0 means output. Default is 1.
+  def Job::verbose=(v) @@verbose = v end
+  # Returns the verbose level. 0 means no output, > 0 means output.
+  def Job::verbose(v) @@verbose end
+
+  # Is the string identifier of the job. Set in #new.
+  attr_reader :name
+  # Is an arbitrary object containing user data for the job. Set in #new.
+  attr_accessor :data
+  # A string with the name of the script/binary to call in order to
+  # execute the job. Set in +run+.
+  attr_reader :host
+  # Will be set when job is run and contains the working directory on the
+  # (remote) host. Set in +run+.
+  # You could change it in the +pre_run_handler+.
+  attr_accessor :working_directory
+  # The parameters for the +client_command+, set in #new.
+  attr_reader :params
+  # The command to be executed on the (remote) machine. Set in #new.
+  # This is either a string or a +Proc+ object (see #new).
+  attr_accessor :client_command
+  # The results object that may be modified in the different handlers.
+  # It is initialized with true. If it is false or nil in the end, the job is
+  # regarded as FAILED, otherwise as SUCCESS.
+  attr_accessor :results
+  # Returns the status of the execution of the job in one of the values:
+  # WAITING, RUNNING, FAILED, SUCCESS
+  attr_accessor :status
+  # A single job object or an array of jobs that must finish before this job can run.
+  # (This means, they need all status == SUCCESS.)
+  attr_accessor :dependencies
+  # Number of times the job failed to run with success
+  attr_reader :numTries
+  def number
+    @number
+  end
+  # list of host names the job failed on
+  attr_reader :failedOnHosts
+
+  # The arguments must be passed as a hash where the keys are the parameter names listed below.
+  # You need not write {}.
+  # +:name+:: is the string identifier of the job
+  # +:params+:: are the parameters for the command
+  # +:data+::
+  #   is an arbitrary object containing user data for the job.
+  #   It is available in the +pre_run_handler+ and can be accessed by the +data+
+  #   attribute whenever a job object is given.
+  # +:dependencies+:: 
+  #   A job or an array of jobs that must be finished first. Default is nil.
+  #   Prevent circular or unfulfillable dependencies.
+  # +:client_command+:: 
+  #   either a string with the name of the command to call in order to
+  #   execute the job on the (remote) machine or a +Proc+ object. In the 
+  #   latter case it receives the job as argument. +Proc+ objects are
+  #   executed on the server only. This should come in handy for some small
+  #   tasks that depend on other jobs but note that it increases the load
+  #   on the server. For +Proc+ commands no output handler will be called.
+  #   A default value can be set for all new jobs by +default_client_command+.
+  # <tt>:pre_run_handler |job|</tt>::
+  #   is a Proc object that is called before the command is run.
+  # <tt>:output_handler |file,job|</tt>::
+  #   is a Proc object that handles the output of
+  #   the command output. You have to read the lines by yourself: (file.gets).
+  #   The default handler puts the file's lines to stdout.
+  #   The user can collect/store the results in the <tt>job.results</tt> variable,
+  #   that is +true+ by default.
+  # <tt>post_run_handler |job|</tt>::
+  #   is a Proc object that is called after the command is run.
+  #
+  # See JobServer for an example.
+  def initialize(args)
+    raise(ArgumentError, "Hash arguments expected.",caller) unless args.kind_of?(Hash)
+    args.each_pair{|key,value| args[key.to_sym]= value} # allow string keys also
+    @@jobNumber += 1
+    @number = @@jobNumber
+    @name = args[:name]
+    @params = args[:params]
+    @data = args[:data]
+    @dependencies = args[:dependencies] || []
+    @dependencies = [@dependencies] unless @dependencies.is_a?(Array)
+    @client_command = args[:client_command] || @@default_client_command
+    raise(ArgumentError, "No command specified for job #{name}", caller) if @client_command == ""
+    @pre_run_handler = args[:pre_run_handler]
+    @output_handler = args[:output_handler] || Proc.new {|file,job| puts file.gets }
+    @post_run_handler = args[:post_run_handler]
+    @host = nil  #hostname where the job is executed (nil if only scheduled)
+    @worker_name = nil
+    @working_directory = ""
+    @numTries = 0
+    @results = nil
+    @filesToRemove = [] #list: (host, filename) of files to be removed after run of job
+    @status = WAITING
+    @failedOnHosts = []
+  end
+
+  # Calls the +client_command+ on the given host.
+  # The job will be executed by a thread +worker_name+ in +working_directory+.
+  # If hostname != "localhost", the command is executed via ssh on the remote
+  # machine.
+  # Returns +@results+.
+  def run(hostname, worker_name, working_directory) # :nodoc:
+    @host = hostname
+    @worker_name = worker_name
+    @working_directory = working_directory || ""
+    @results = true # default result, can be changed by user
+    @status = RUNNING
+    @numTries += 1
+    @pre_run_handler.call(self) if @pre_run_handler
+    if @client_command.is_a?(Proc)
+      @host = "localhost"
+      @client_command.call(self) #run the Proc on the server (ignore that it should run remotely)
+    else
+      nice = if @@nicelevel and not (PLATFORM.downcase =~ /win/)
+               "nice -#{@@nicelevel}"
+             else
+               "" # don't use nice
+             end
+      chdir = (@working_directory != "") ? "cd #{quote(@working_directory)};" : ""
+      cmd = "#{chdir}#{nice} #{@client_command} #{@params}"
+      cmd = "ssh #{hostname} #{quote(cmd)}" if hostname != "localhost"
+      runCommand(cmd)
+    end
+    @post_run_handler.call(self) if @post_run_handler
+    failedOnHosts |= [hostname] unless @results
+    @status = @results ? Job::SUCCESS : Job::FAILED
+    return @results
+  end
+
+  # Do you want to store the data of an object in a file
+  # that will be accessible to the command? If yes, then use this method.
+  # +data+:: is an object that can be written with puts
+  # +filename+:: is relative to +working_directory+
+  # +temporary+:: determines if the file should be removed after the job has run.
+  # Note: you could use this method in +pre_run_handler+ even to create the client
+  # script before executing it.
+  def store_data(data, filename, temporary = true)
+    filename = File.join(@working_directory, filename)
+    if @host == "localhost" then
+      File.open(filename,"w") { |file| file.puts data }
+    else
+      tempFile = Tempfile.new("#{File.basename(filename)}.tmp")
+      tempFile.puts data
+      tempFile.close
+      copyToHost(tempFile.path, filename)
+      tempFile.close(true) # remove local temp file
+    end
+    @filesToRemove << filename if temporary
+  end
+
+  # Returns the current state of the job.
+  def to_s
+    s = "#{@name}: "
+    case status
+    when WAITING
+      s += "waiting"
+    when RUNNING
+      s += "running"
+    when SUCCESS
+      s += "finished"
+    when FAILED
+      s += "failed"
+    end
+    s += ", try #{@numTries}" if @numTries > 1
+    return s
+  end
+
+  # This is called by the jobserver in order to decide if the job should be run
+  # on the given host. Default behaviour is true (run on all hosts).
+  # You can override this function, e.g.:
+  #   require 'jobserver'
+  #
+  #   class Job
+  #     def runsOnHost(hostname)
+  #       case hostname
+  #       when ...
+  #       end
+  #     end
+  #   end
+  # Beware not to construct dead-lock situations when no host satisfies a constraint.
+  def runsOnHost(hostname)
+    true
+  end
+
+  # Returns a subset of the given hosts for which #runsOnHost is +true+.
+  def runsOnHosts(hosts)
+    hosts.select {|host| runsOnHost(host)}
+  end
+
+  protected
+  # Copy files to the remote(!) host.
+  # Perhaps a file will have to be created on the fly by the job creator.
+  # This method can be used e.g. in the +pre_run_handler+ if files are needed
+  # on the remote host for the command to work on.
+  # +source+ are the files on the local host
+  # +destination+ the destination directory on @host, relative to
+  # @working_directory. Used by +store_data+.
+  def copyToHost(source, destination)
+    system("scp -q #{source} #{@host}:#{@working_directory}/#{destination}")
+  end
+
+  protected
+  # Called by +run+
+  def runCommand(command)
+    puts "Running job ##{@number} on #{@worker_name}: #{command}" if @@verbose > 0
+    IO.popen(command, "r") do |cin|
+      until cin.eof?
+        if @output_handler
+          @output_handler.call(cin, self)
+        else
+          cin.gets # read the line
+        end
+      end
+    end
+    #remove temporary files
+    unless @filesToRemove.empty?
+      if @host == "localhost" then File.delete(*@filesToRemove)
+      else system("ssh #{@host} rm #{@filesToRemove.join(' ')}")
+      end
+    end
+    @worker_name = nil # mark the job as finished
+  end
+
+  # Quotes all shell-special characters in the command.
+  def quote(command)
+    command.gsub(/[|&;()\\<>'"]/) {|x| '\\'+x}
+  end
+end
+
+###############################################################################
+
+# This class is used internally by the JobServer to collect statistics for
+# each host.
+class HostStatistics # :nodoc:
+  attr_accessor :num_jobs_finished
+  def initialize
+    @num_jobs_started = 0
+    @num_jobs_finished = 0
+    @num_jobs_failed = 0
+    @userTime = 0.0
+    @startTime = 0.0
+  end
+
+  # is called before a job is run 
+  def begin_update
+    @startTime = Time.new
+    @num_jobs_started += 1
+  end
+
+  # is called when a job has (successfully) terminated
+  def end_update(success)
+    @userTime += Time.new - @startTime
+    if success
+      @num_jobs_finished += 1
+    else
+      @num_jobs_failed += 1
+    end
+  end
+
+  # returns the current statistics
+  def to_s
+    avgTime = @num_jobs_finished > 0 ? '%.1f s' % (@userTime / @num_jobs_finished) : '<no jobs finished>'
+    s="Number of jobs finished: #{@num_jobs_finished}, average time per job: #{avgTime}"
+    if (diff=(@num_jobs_started-@num_jobs_failed-@num_jobs_finished)) > 0
+      s += ", #{diff} job#{(diff > 1) ? 's' : ''} running" 
+    end
+    s += ", #{@num_jobs_failed} job#{(@num_jobs_failed > 1) ? 's' : ''} failed" if @num_jobs_failed > 0
+    s
+  end
+end
+
+###### Job Server #############################################################
+
+
+# == Usage example:
+#
+# === Create job handlers
+#  require 'jobserver'
+#  pre_run_handler = Proc.new do |job|
+#    puts "Running job #{job.name} on #{job.host}"
+#    # Initialize the results object as an empty array:
+#    job.results = []
+#  end
+#  output_handler = Proc.new do |file, job|
+#    line = file.gets
+#    job.results << $1 if line =~ /result: (.*)/
+#  end
+#  post_run_handler = Proc.new do |job|
+#    if job.results.empty?
+#      puts "Error executing job #{job.name} on #{job.host}.\n\t#{job}"
+#    else
+#      puts $result = job.results.join(",")
+#    end
+#  end
+# === Create the jobs
+#  Job.default_client_command = "runclient"
+#  myJobQueue = []
+#  10.times{|i| myJobQueue << Job.new(:name=>"job#{i}", :params=>"#{i}", :pre_run_handler => pre_run_handler, 
+#                                     :output_handler=>output_handler, :post_run_handler=>post_run_handler)}
+# === Create the server
+#  server = JobServer.new(myJobQueue, "~/work") #run 1 local worker implicitly
+#  server.add_ssh_worker("192.168.0.1", "~/work_sparc")
+#  server.add_ssh_worker("192.168.0.2", "~/work", 2)
+#  server.dumpStatistics
+#  server.serve # Wait until all jobs have finished
+#
+class JobServer
+  class Deadlock < Exception;  end
+  # an array of the worker threads currently running
+  attr_reader :workers
+  # <tt>hostStats[hostname]</tt> returns the +HostStatistics+ object for the string +hostname+
+  attr_reader :hostStats
+
+  # Instantiates a new JobServer object and creates the given number of local clients,
+  # called _workers_.
+  # +jobQueue+::
+  #   is an array of jobs of type Job.
+  #   Jobs on the _left_ side of the array are processed first.
+  #   Jobs that had errors during execution will be enqueued at the end of the queue.
+  #   If you want to decide whether to re-enqueue a job that had errors you can replace
+  #   the method #retryJob by your own.
+  # +local_working_directory+:: is the directory in which local clients are launched
+  # +numLocalWorkers+::
+  #   is the number of client workers which run on the server itself (e.g. number of CPUs)
+  # +terminateWorkersWhenJobQueueEmpty+::
+  #   false means, the workers continue to run, even if the queue is empty
+  #   until #close was called. This can be used if you want to add jobs while
+  #   others are running. The situation 
+  #   can occur that all remaining jobs are running and the queue is empty but
+  #   you want to add more jobs.
+  def initialize(jobQueue, local_working_directory = "", numLocalWorkers = 1, terminateWorkersWhenJobQueueEmpty = true)
+    @jobQueue = jobQueue
+    @jobQueue.extend(MonitorMixin)
+    @initialQueueLength = @jobQueue.length
+    @jobsRunning = []
+    @jobsRunning.extend(MonitorMixin)
+    @noJobs_cond = @jobQueue.new_cond
+    @local_working_directory = local_working_directory
+    @workers = []
+    @hostStats = Hash.new
+    @hostStats.extend(MonitorMixin)
+    @terminateWorkersWhenJobQueueEmpty = terminateWorkersWhenJobQueueEmpty
+    @usedHosts = []
+    add_local_worker(numLocalWorkers)
+  end
+
+  # serve waits for all jobs to terminate and outputs statistics if verbose is true
+  def serve(verbose = true)
+    raise(Exception, "No workers registered but serve was called. The jobs can't be processed!",caller) if @workers.empty?
+    @workers.each{|worker| worker.wakeup}  #wake all workers up
+    @workers.each{|worker| worker.join}  #wait for all workers to finish
+
+    @dumpStatThread.wakeup if defined?(@dumpStatThread) and @dumpStatThread.status #wake up the statistics dumper
+
+    #output statistics
+    if verbose
+      puts "Host statistics:\n================"
+      @hostStats.to_a.sort{|x,y| x[0]<=>y[0]}.each{|host,stats| puts "#{host}: #{stats}"}
+      puts
+    end
+  end
+
+  # close must be called only when you have set terminateWorkersWhenJobQueueEmpty
+  # to false during instantiation. Then you tell the server that no further
+  # jobs will be added to the job queue.
+  # All worker threads that have been waiting for new jobs will terminate now.
+  # You should still wait for all workers to complete. Use *serve* to do so.
+  def close
+    @terminateWorkersWhenJobQueueEmpty = true
+    @noJobs_cond.signal
+  end
+
+  # Writes statistics for each host to the given file.
+  # If no directory is given then in the local working directory given to
+  # +new+ is used.
+  # +timeInSec+ is the time after which the current statistics will
+  # be written. Time set to 0 means, write only when all jobs have finished.
+  # When the jobserver terminates, the last state will be written however the time.
+  def dumpStatistics(filename = "jobserver_stats.txt", timeInSec=60)
+    filename = File.join(@local_working_directory, filename) if filename == File.basename(filename)
+    
+    @dumpStatThread = Thread.new(timeInSec) do |sleepTime|
+      loop do
+        sleep(sleepTime)
+        File.open(filename,"w") do |file|
+          file.puts "Host statistics:\n================"
+          @hostStats.synchronize do
+            @hostStats.to_a.sort{|x,y| x[0]<=>y[0]}.each{|host,stats| file.puts "#{host}: #{stats}"}
+          end
+          unless @jobsRunning.empty? 
+            @jobsRunning.synchronize do
+              file.puts "\nJobs running:\n============"
+              file.puts @jobsRunning.map {|job| "#{job.host}: #{job}" },""
+            end
+          end
+          unless @jobQueue.empty?
+            @jobQueue.synchronize do 
+              file.puts s="Jobs in the queue: (#{@jobQueue.length}/#@initialQueueLength remaining)"
+              file.puts "="*s.length                       
+              file.puts @jobQueue
+            end
+          end
+        end
+      end 
+    end
+  end
+  
+  # appends a job at the end of the queue and informs the sleeping worker
+  # threads that new jobs are available.
+  def add_job(job)
+    @jobQueue << job
+    @noJobs_cond.broadcast #wake up workers that wait for new jobs
+  end
+  alias :<< :add_job
+
+  # Adds a worker thread that processes jobs on the local machine.
+  # This method is called automatically on object instantiation.
+  # +numWorkers+ indicates, how many workers should use this client. It should usually not 
+  # exceed the number of CPUs that are available for the host.
+  # You may wish to set it to zero, if the server machine should not be used as a client 
+  # as well.
+  def add_local_worker(numWorkers = 1)
+    add_worker("localhost", @local_working_directory, numWorkers)
+  end
+
+  # Adds a worker thread that processes jobs on a given remote host machine.
+  # +hostname+ can contain a username like: <tt>fool@192.168.0.1</tt>
+  # If +working_directory+ is not empty then the client command will be executed in the
+  # given directory.
+  # +numWorkers+ indicates, how many workers should use this client. It should usually not 
+  # exceed the number of CPUs that are available for the host.
+  def add_ssh_worker(hostname, working_directory = "", numWorkers = 1)
+    hostname = "localhost" if ENV['HOSTNAME'] == hostname
+    add_worker(hostname, working_directory, numWorkers)
+  end
+
+  # Is called when +job+ couldn't be executed. Determines whether to try to rerun the job.
+  # Whether a job had errors or not depends on whether the +results+ object is false/nil
+  # or not. See #Job.new for further details about +results+.
+  # By default, +retryJob+ allows three tries until the job is given up.
+  #  If you want another behaviour, override the method +retryJob+ by your own, e.g.:
+  #   require 'jobserver'
+  #
+  #   class JobServer
+  #     def retryJob(job)
+  #       puts "Job failed: #{job.name}"
+  #       return false
+  #     end
+  #   end
+  def retryJob(job)
+    if job.numTries < 3
+      puts "FAILURE: Will try to run job later: #{job}"
+      true
+    end # else return false implicitly
+  end
+
+  # Removes and returns a Job from the queue using a FIFO strategy but
+  # considering the dependency constraints of the jobs. That means, only
+  # those jobs are chosen that have only jobs in their dependency list that 
+  # are finished. The user has to guarantee that the dependency-graph has no
+  # cycles.
+  # Furthermore each job has a function :runsOnHost that you might want to
+  # override in order to decide whether the job is fit to run on the given host.
+  # It may happen that no host gets a job. This may result in a job that failed but
+  # that is cruical to the execution of others so that these cannot be executed any
+  # more. This deadlock situation raises an exception.
+  #
+  # When a job failed on a host, it tries to rerun on other hosts first (if there are any).
+  def getNextJob(hostname)
+    job = @jobQueue.find do |job|
+      # it is assumed that all jobs form the queue are WAITING
+      ok = (job.dependencies.length == job.dependencies.select{|dep| dep.status == Job::SUCCESS}.length and
+              job.runsOnHost(hostname))
+      # ok == true => job could be run on 'hostname'
+      # did the job fail on this host?
+      if ok and job.failedOnHosts.include?(hostname)
+        # if it failed already on some hosts but there are still machines it could run on and didn't fail on them yet,
+        # so run on those hosts first. If otherwise the job failed on all hosts, try the current host again.
+        ok = job.runsOnHosts(@usedHosts - job.failedOnHosts).empty?
+      end
+      ok
+    end
+    return @jobQueue.delete(job) # returns nil if job == nil
+  end
+
+  private
+  # Creates a new worker thread. This method is called by +add_local_worker+ and +add_ssh_worker+.
+  # The worker is put into a sleep state and will be woken up at Jobserver.serve
+  def add_worker(hostname, working_directory, numWorkers)
+    @usedHosts |= [hostname]
+    numWorkers.times do |i|
+      worker_name = hostname + ((numWorkers > 1) ? "_#{i}" : "")
+      @hostStats[worker_name] = HostStatistics.new
+      @workers << Thread.new do
+        Thread.current[:getNoJob] = false
+        Thread.stop
+        begin
+          until @jobQueue.empty? do
+            job = nil
+            results = nil
+            #sleep/wait until new jobs arrived or some jobs finished
+            @jobQueue.synchronize { @noJobs_cond.wait_while { 
+                #while queue is empty and we don't want to terminate yet or
+                #there are not jobs that can run right now
+                dowait = ((@jobQueue.empty? and !@terminateWorkersWhenJobQueueEmpty) or 
+                            (!@jobQueue.empty? and (job = getNextJob(hostname)) == nil))
+                Thread.current[:getNoJob] = (job == nil)
+                if dowait
+                  # Before we go to sleep, check:
+                  # The queue is not empty and no worker gets a job? This means: deadlock!
+                  if !@jobQueue.empty? and 
+                      @workers.inject(true) {|prod,worker| prod and worker[:getNoJob] } # true, if every worker has getNoJob==true
+                    Thread.critical = true # halt the other threads during output
+                    #the deadlock can be caused by unsatisfiable or circular dependencies
+                    $stderr.puts "\nDeadlock! -- Current Job Queue (#{@jobQueue.length} jobs):\n========================================="
+                    $stderr.puts @jobQueue
+                    $stderr.puts "\nJobs with unsatisfied dependencies:"
+                    for job in @jobQueue do
+                      $stderr.puts "#{job.name}:"
+                      unsatisfied = job.dependencies.select{|j| j.status != Job::SUCCESS }.join('\n   ')
+                      if unsatisfied == ""
+                        $stderr.puts "  no unsatisfied jobs"
+                        #job.dependencies.each{|j| puts "  "+j.to_s}
+                      else
+                        $stderr.puts "  "+unsatisfied.to_s
+                      end
+                    end
+                    $stderr.puts
+                    $stderr.puts "Before debugging the jobserver please verify, if"
+                    $stderr.puts " a) no job on which a job form the job queue depends on failed"
+                    $stderr.puts " b) the jobs form the job queue cannot be run on any host (if you redefine Job.runsOnHost)"
+                    # $stderr.puts "No worker can get a job because of unsatisfied dependencies or host constraints."
+                    Thread.critical = false
+                    raise Deadlock, "JobServer deadlock: no worker can get a job but there are jobs in the queue."
+                    exit
+                  end
+                end
+                dowait
+              } }
+            if job # did we get a job? No could mean the queue is empty and we want to terminate
+              @jobsRunning.synchronize { @jobsRunning << job }
+              @hostStats.synchronize { @hostStats[worker_name].begin_update }
+              results = job.run(hostname, worker_name, working_directory)
+              @jobsRunning.synchronize { @jobsRunning.delete(job) }
+              #wake up workers that wait for new jobs, because some constraint/dependency could be satisfied now
+              @jobQueue.synchronize { @noJobs_cond.broadcast }
+
+              @hostStats.synchronize { @hostStats[worker_name].end_update(results) }
+              if results then
+                job.dependencies = nil # free them for the garbage collection
+              elsif retryJob(job)
+                job.status = Job::WAITING
+                @jobQueue.synchronize { @jobQueue << job }
+              end
+            end
+          end
+        end until @terminateWorkersWhenJobQueueEmpty
+      end
+    end
+  end
+end