batch-kit 0.3

data/lib/batch-kit/helpers/process.rb

@@ -0,0 +1,101 @@
+require 'shellwords'
+require 'open3'
+
+require_relative '../logging'
+
+
+class BatchKit
+
+    module Helpers
+
+        # Provides support for running an external process.
+        # This support consists of support for:
+        # - launching the process as a child
+        # - capturing the output of the process and logging it
+        # - handling the return code of the process, and raising an exception for
+        #   failures.
+        module Process
+
+            # Provides a means for executing a command-line.
+            #
+            # @param cmd_line [String] The command-line that is to be launched.
+            # @param options [Hash] An options hash.
+            # @option options [Proc] :callback If specified, the supplied Proc will
+            #   be invoked for each line of output produced by the process.
+            # @option options [Proc] :input If specified, the supplied Proc will
+            #   be invoked for each line of output produced by the process. It will
+            #   be passed the pipe on which input for the process can be written,
+            #   plus the last line of output produced. This is useful in cases where
+            #   it is necessary to communicate with the child process via its STDIN.
+            # @return [Fixnum] The exit status code from the external process.
+            def popen(cmd_line, options = {}, &block)
+                callback = options[:callback]
+                input = options[:input]
+                IO.popen(cmd_line, input ? 'r+' : 'r') do |pipe|
+                    while !pipe.eof?
+                        line = pipe.gets.chomp
+                        input.call(pipe, line) if input
+                        callback.call(line) if callback
+                        block.call(line) if block_given?
+                    end
+                end
+                $?.exitstatus
+            end
+            module_function :popen
+
+
+            # Launch an external process with logging etc. By default, an exception
+            # will be raised if the process returns a non-zero exit code.
+            #
+            # @param cmd_line [String, Array<String>] The command-line to be run,
+            #   in the form of either a single String, or an Array of Strings.
+            # @param options [Hash] An options hash.
+            # @option options [Boolean] :raise_on_error If true (default), an
+            #   exception is raised if the return code is not a success code.
+            # @option options [Fixnum, Array<Fixnum>] The return code(s) that the
+            #   process can return if successful (default 0).
+            # @option options [Boolean] :show_duration If true (default), logs the
+            #   duration taken by the process.
+            # @option options [Logger] :logger The logger to use; defaults to using
+            #   a logger named after the process being executed.
+            def launch(cmd_line, options = {}, &block)
+                exe = cmd_line.is_a?(String) ?
+                    File.basename(Shellwords.shellwords(cmd_line.gsub(/\\/, '/')).first) :
+                    File.basename(cmd_line.first)
+
+                raise_on_error = options.fetch(:raise_on_error, true)
+                show_duration = options.fetch(:show_duration, true)
+                success_code = options.fetch(:success_code, 0)
+                log = options.fetch(:logger, BatchKit::LogManager.logger(exe))
+                log_level = options.fetch(:log_level, :detail)
+                unless block_given? || options[:callback]
+                    options = options.dup
+                    options[:callback] = lambda{ |line| log.send(log_level, line) }
+                end
+
+                log.trace("Executing command line: #{cmd_line}") if log
+                begin
+                    start = Time.now
+                    rc = popen(cmd_line, options, &block)
+                ensure
+                    if log && show_duration
+                        log.detail "#{exe} completed in #{Time.now - start} seconds with exit code #{rc}"
+                    end
+                end
+
+                if raise_on_error
+                    ok = case success_code
+                    when Fixnum then success_code == rc
+                    when Array then success_code.include?(rc)
+                    end
+                    raise "#{exe} returned failure exit code #{rc}" unless ok
+                end
+                rc
+            end
+            module_function :launch
+
+        end
+
+    end
+
+end

data/lib/batch-kit/helpers/zip.rb

@@ -0,0 +1,30 @@
+require 'fileutils'
+require 'zip/zip'
+
+
+class BatchKit
+
+    module Helpers
+
+        module Zip
+
+            # Creates a new +zip_file+, adding +files+ to it.
+            #
+            # @param zip_file [String] A path to the zip file to be created
+            # @param files [String] One or more paths to files to be added to
+            #   the zip.
+            def create_zip(zip_file, *files)
+                FileUtils.rm_f(zip_file)
+                Zip::ZipFile.open(zip_file, Zip::ZipFile::CREATE) do |zip|
+                    files.each do |file|
+                        zip.add(File.basename(file), file)
+                    end
+                    yield zip if block_given?
+                end
+            end
+
+        end
+
+    end
+
+end

data/lib/batch-kit/job.rb

@@ -0,0 +1,11 @@
+require_relative 'events'
+require_relative 'lockable'
+require_relative 'framework/definable'
+require_relative 'framework/job_definition'
+require_relative 'framework/task_definition'
+require_relative 'framework/runnable'
+require_relative 'framework/job_run'
+require_relative 'framework/task_run'
+require_relative 'framework/acts_as_job'
+require_relative 'framework/job'
+

data/lib/batch-kit/lockable.rb

@@ -0,0 +1,138 @@
+require 'timeout'
+require_relative 'events'
+
+
+class BatchKit
+
+    # Defines lockable behaviour, which can be added to any batch process.
+    # This behavior allows a process to define a named lock that it needs
+    # exclusively during execution.
+    # When the process is about to be executed, it will first attempt to obtain
+    # the named lock. If it is successful, execution will proceed as normal, and
+    # on completion of processing (whether succesful or otherwise), the lock
+    # will be released.
+    # If the lock is already held by another process, the requesting process
+    # will block and wait for the lock to become available. The process will
+    # only wait as long as lock_wait_timeout; if the lock has not become
+    # availabe in that time period, a LockTimeout exception will be thrown,
+    # and processing will not take place.
+    module Lockable
+
+        # Attempts to obtain the named lock +lock_name+. If the lock is already
+        # held by another process, this method blocks until one of the following
+        # occurs:
+        # - the lock is released by the process that currently holds it
+        # - the lock expires, by reaching it's timeout period
+        # - the +lock_wait_timeout+ period is reached.
+        #
+        # Lock management is managed via the event publishing system; subscribers
+        # to the 'lock?' event indicate whether a lock is available by their
+        # response to the event. A value of false indicates the lock is
+        # currently held; a response of true indicates the lock has been granted.
+        #
+        # @param lock_name [String] The name of the lock that is needed.
+        # @param lock_timeout [Fixnum] The maximum number of seconds that this
+        #   process can hold the requested lock before it times out (allowing
+        #   any other processes waiting on the lock to proceed). This value
+        #   should be set high enough that the lock does not timeout while
+        #   processing that relies on the lock is not still running.
+        # @param lock_wait_timeout [Fixnum] The maximum time this process is
+        #   prepared to wait for the lock to become available. If not specified,
+        #   the wait will timeout after the same amount of time as +lock_timeout+.
+        # @raise Timeout::Error If the lock is not obtained within
+        #   +lock_wait_timeout+ seconds.
+        def lock(lock_name, lock_timeout, lock_wait_timeout = nil)
+            unless lock_timeout && lock_timeout.is_a?(Fixnum) && lock_timeout > 0
+                raise ArgumentError, "Invalid lock_timeout; must be > 0"
+            end
+            unless lock_wait_timeout.nil? || (lock_wait_timeout.is_a?(Fixnum) && lock_wait_timeout >= 0)
+                raise ArgumentError, "Invalid lock_wait_timeout; must be nil or >= 0"
+            end
+            unless Events.has_subscribers?(self, 'lock?')
+                if self.respond_to?(:log)
+                    log.warn "No lock manager available; proceeding without locking"
+                end
+                return
+            end
+            lock_wait_timeout ||= lock_timeout
+            lock_expire_time = nil
+            wait_expire_time = Time.now + lock_wait_timeout
+            if lock_wait_timeout > 0
+                # Loop waiting for lock if not available
+                begin
+                    Timeout.timeout(lock_wait_timeout) do
+                        i = 0
+                        loop do
+                            lock_holder = {}
+                            lock_expire_time = Events.publish(self, 'lock?', lock_name,
+                                                                     lock_timeout, lock_holder)
+                            break if lock_expire_time
+                            if i == 0
+                                Events.publish(self, 'lock_held', lock_name,
+                                                      lock_holder[:lock_holder],
+                                                      lock_holder[:lock_expires_at])
+                                Events.publish(self, 'lock_wait', lock_name, wait_expire_time)
+                            end
+                            sleep 1
+                            i += 1
+                        end
+                        Events.publish(self, 'locked', lock_name, lock_expire_time)
+                    end
+                rescue Timeout::Error
+                    Events.publish(self, 'lock_wait_timeout', lock_name, wait_expire_time)
+                    raise Timeout::Error, "Timed out waiting for lock '#{lock_name}' to become available"
+                end
+            else
+                # No waiting for lock to become free
+                lock_holder = {}
+                if lock_expire_time = Events.publish(self, 'lock?', lock_name, lock_timeout, lock_holder)
+                    Events.publish(self, 'locked', lock_name, lock_expire_time)
+                else
+                    Events.publish(self, 'lock_held', lock_name,
+                                          lock_holder[:lock_holder], lock_holder[:lock_expires_at])
+                    Events.publish(self, 'lock_wait_timeout', lock_name, wait_expire_time)
+                    raise Timeout::Error, "Lock '#{lock_name}' is already in use"
+                end
+            end
+        end
+
+
+        # Release a lock held by this object.
+        #
+        # @param lock_name [String] The name of the lock to be released.
+        def unlock(lock_name)
+            unless Events.has_subscribers?(self, 'unlock?')
+                return
+            end
+            if Events.publish(self, 'unlock?', lock_name)
+                Events.publish(self, 'unlocked', lock_name)
+            end
+        end
+
+
+        # Obtains the requested +lock_name+, then yields to the supplied block.
+        # Ensures the lock is released when the block ends or raises an error.
+        #
+        # @param lock_name [String] The name of the lock to obtain.
+        # @param lock_timeout [Fixnum] The maximum number of seconds that this
+        #   process can hold the requested lock before it times out (allowing
+        #   any other processes waiting on the lock to proceed). This value
+        #   should be set high enough that the lock does not timeout while
+        #   processing that relies on the lock is not still running.
+        # @param lock_wait_timeout [Fixnum] The maximum time this process is
+        #   prepared to wait for the lock to become available. If not specified,
+        #   the wait will timeout after the same amount of time as +lock_timeout+.
+        # @raise Timeout::Error If the lock is not obtained within
+        #   +lock_wait_timeout+ seconds.
+        def with_lock(lock_name, lock_timeout, lock_wait_timeout = nil)
+            self.lock(lock_name, lock_timeout, lock_wait_timeout)
+            begin
+                yield
+            ensure
+                self.unlock(lock_name)
+            end
+        end
+
+    end
+
+end

data/lib/batch-kit/loggable.rb

@@ -0,0 +1,78 @@
+require_relative 'logging'
+
+
+class BatchKit
+
+    # Adds logging behaviour to a batch-kit process, causing its lifecycle to be
+    # logged.
+    module Loggable
+
+        # Returns a logger instance named after the class
+        def log
+            @log ||= LogManager.logger(self.class.name)
+        end
+
+
+        if defined?(BatchKit::Events)
+
+            # Subscribe to batch-kit lifecycle events that should be logged
+            Events.subscribe(Configurable, 'config.post-load') do |job_cls, cfg|
+                if cfg.has_key?(:log_level) || cfg.has_key?(:log_file)
+                    log = LogManager.logger(job_cls.name)
+                    if cfg[:log_level]
+                        log.level = cfg[:log_level]
+                        log.config "Log level set to #{cfg[:log_level].upcase}"
+                    end
+                    if cfg.has_key?(:log_file)
+                        log.config "Logging output to: #{cfg[:log_file]}" if cfg[:log_file]
+                        FileUtils.mkdir_p(File.dirname(cfg[:log_file]))
+                        log.log_file = cfg[:log_file]
+                    end
+                end
+            end
+            Events.subscribe(Loggable, 'sequence_run.execute') do |job_obj, run, *args|
+                job_obj.log.info "Sequence '#{run.label}' started"
+            end
+            Events.subscribe(Loggable, 'job_run.execute') do |job_obj, run, *args|
+                id = run.job_run_id ? " as job run #{run.job_run_id}" : ''
+                job_obj.log.info "Job '#{run.label}' started on #{run.computer} by #{run.run_by}#{id}"
+            end
+            Events.subscribe(Loggable, 'task_run.execute') do |job_obj, run, *args|
+                id = run.task_run_id ? " as task run #{run.task_run_id}" : ''
+                job_obj.log.info "Task '#{run.label}' started#{id}"
+            end
+            %w{sequence_run job_run task_run}.each do |runnable|
+                Events.subscribe(Loggable, "#{runnable}.post-execute") do |job_obj, run, ok|
+                    job_obj.log.info "#{run.class.name.split('::')[-2]} '#{run.label}' completed #{
+                        ok ? 'successfully' : 'with errors'} in #{'%.3f' % run.elapsed} seconds"
+                end
+            end
+
+            Events.subscribe(Lockable, 'lock_wait') do |job_run, lock_name|
+                if (job_obj = job_run.object).is_a?(Loggable)
+                    job_obj.log.detail "Waiting for lock '#{lock_name}' to become avaialable"
+                end
+            end
+            Events.subscribe(Lockable, 'lock_held') do |job_run, lock_name, lock_holder, lock_expire_time|
+                if (job_obj = job_run.object).is_a?(Loggable)
+                    job_obj.log.warn "Lock '#{lock_name}' is currently held by #{lock_holder}; expires at #{
+                        lock_expire_time.strftime('%H:%M:%S')}"
+                end
+            end
+            Events.subscribe(Lockable, 'locked') do |job_run, lock_name, lock_expire_time|
+                if (job_obj = job_run.object).is_a?(Loggable)
+                    job_obj.log.detail "Obtained lock '#{lock_name}'; expires at #{
+                        lock_expire_time.strftime('%H:%M:%S')}"
+                end
+            end
+            Events.subscribe(Lockable, 'unlocked') do |job_run, lock_name|
+                if (job_obj = job_run.object).is_a?(Loggable)
+                    job_obj.log.detail "Released lock '#{lock_name}'"
+                end
+            end
+
+        end
+
+    end
+
+end

data/lib/batch-kit/logging.rb

@@ -0,0 +1,169 @@
+require 'fileutils'
+
+
+class BatchKit
+
+    module Logging
+
+        # Log levels available
+        LEVELS = [:error, :warning, :info, :config, :detail, :trace, :debug]
+
+        # Supported logging frameworks
+        FRAMEWORKS = [
+            :null,
+            :stdout,
+            :log4r,
+            :java_util_logging
+        ]
+
+        # Method aliasing needed to provide log methods corresponding to levels
+        FRAMEWORK_INIT = {
+            null: lambda{
+                require_relative 'logging/null_logger'
+            },
+            stdout: lambda{
+                require_relative 'logging/stdout_logger'
+            },
+            java_util_logging: lambda{
+                require_relative 'logging/java_util_logger'
+            },
+            log4r: lambda{
+                require_relative 'logging/log4r_logger'
+            }
+        }
+
+    end
+
+
+    # Used for setting the log framework to use, and retrieving a logger
+    # from the current framework.
+    class LogManager
+
+        class << self
+
+            def configure(options = {})
+                self.log_framework = options[:log_framework] if options[:log_framework]
+                if options.fetch(:log_color, true)
+                    case self.log_framework
+                    when :log4r
+                        require 'color_console/log4r_logger'
+                        Console.replace_console_logger(logger: 'batch')
+                    when :java_util_logging
+                        require 'color_console/java_util_logger'
+                        Console.replace_console_logger(
+                            level: Java::JavaUtilLogging::Level::FINE,
+                            level_labels: {
+                                Java::JavaUtilLogging::Level::FINE => 'DETAIL',
+                                Java::JavaUtilLogging::Level::FINER => 'TRACE'
+                            })
+                    else
+                        require 'color_console'
+                    end
+                end
+            end
+
+
+            # Returns a symbol identifying which logging framework is being used.
+            def log_framework
+                unless @log_framework
+                    if RUBY_PLATFORM == 'java'
+                        LogManager.log_framework = :java_util_logging
+                    else
+                        begin
+                            require 'log4r'
+                            LogManager.log_framework = :log4r
+                        rescue LoadError
+                            LogManager.log_framework = :stdout
+                        end
+                    end
+                end
+                @log_framework
+            end
+
+
+            # Sets the logging framework
+            def log_framework=(framework)
+                unless Logging::FRAMEWORKS.include?(framework)
+                    raise ArgumentError, "Unknown logging framework #{framework.inspect}"
+                end
+                if @log_framework
+                    lvl = self.level
+                end
+                @log_framework = framework
+                if init_proc = Logging::FRAMEWORK_INIT[@log_framework]
+                    init_proc.call
+                end
+                self.level = lvl if lvl
+                logger.trace "Log framework is #{@log_framework}"
+            end
+
+
+            # Returns the current root log level
+            def level
+                logger.level
+            end
+
+
+            # Sets the log level
+            def level=(level)
+                case log_framework
+                when :log4r
+                    lvl = Log4r::LNAMES.index(level.to_s.upcase)
+                    Log4r::Logger.each_logger{ |l| l.level = lvl }
+                else
+                    logger.level = level
+                end
+            end
+
+
+            # Returns a logger with a given name, which must be under the 'batch'
+            # namespace. If name is omitted, the logger is named 'batch'. If a
+            # name is specified that is not under 'batch', then it is prepended
+            # with 'batch'.
+            #
+            # @return [Logger] a logger object that can be used for generating
+            #   log messages. The type of logger returned will depend on the
+            #   log framework being used, but the logger is guaranteed to
+            #   implement the following log methods:
+            #   - error
+            #   - warning
+            #   - info
+            #   - config
+            #   - detail
+            #   - trace
+            #   - debug
+            def logger(name = nil)
+                case name
+                when NilClass, ''
+                    name = 'batch'
+                when /^batch/
+                when /\./
+                when String
+                    name = "batch.#{name}"
+                end
+                case log_framework
+                when :stdout
+                    BatchKit::Logging::StdOutLogger.logger(name)
+                when :java_util_logging
+                    BatchKit::Logging::JavaLogFacade.new(Java::JavaUtilLogging::Logger.getLogger(name))
+                when :log4r
+                    log4r_name = name.gsub('.', '::')
+                    BatchKit::Logging::Log4rFacade.new(Log4r::Logger[log4r_name] ||
+                                                    Log4r::Logger.new(log4r_name))
+                else BatchKit::Logging::NullLogger.instance
+                end
+            end
+
+        end
+
+
+        if defined?(Events) && defined?(Configurable)
+            Events.subscribe(Configurable, 'post-configure') do |src, cfg|
+                LogManager.configure(cfg)
+            end
+        end
+
+    end
+
+end
+