batch-kit 0.3

data/lib/batch-kit/framework/definable.rb

@@ -0,0 +1,169 @@
+class BatchKit
+
+    # Captures details of a definable batch process, e.g. a Task or Job.
+    #
+    # @abstract
+    class Definable
+
+        class << self
+
+            # Register additional properties to be recorded on this definable.
+            #
+            # @param props [Array<Symbol>] The names of properties to be added
+            #   to the definition. Used by sub-classes to add to the available
+            #   properties. This provides a mechanism by which associated
+            #   Run objects can obtain a list of process properties that they
+            #   can delegate.
+            def add_properties(*props)
+                attr_accessor(*props)
+                properties.concat(props)
+            end
+
+
+            # @return [Array<Symbol>] the names of properties available on this
+            #   definition.
+            def properties
+                @properties ||= []
+            end
+
+
+            # When this class is inherited, we need to copy the common property
+            # names into the sub-class, since each sub-class needs the common
+            # property names defined on this base class, as well as any sub-
+            # class specific properties.
+            def inherited(subclass)
+                subclass.instance_variable_set(:@properties, @properties.clone)
+            end
+
+        end
+
+
+        # @!attribute :name [String] A user-friendly name for this process.
+        # @!attribute :description [String] A short description of what this
+        #   process does.
+        # @!attribute :instance [String] An optional expression used to assign
+        #   an instance identifier to a process.
+        #   An instance identifier allows a process that has different execution
+        #   profiles (typically depending on the arguments it is run with)
+        #   to identify which of those profiles it is executing. This expression
+        #   will be evaluated at the time this process is invoked, and the
+        #   result will become the instance identifier for the Runnable.
+        # @!attribute :runs [Array<Runnable>] Array of runs of this process.
+        # @!attribute :lock_name [String] The name of any lock that this
+        #   process requires before it can proceed. If nil, no lock is
+        #   required and the process can commence without any co-ordination
+        #   with other processes.
+        # @!attribute :lock_timeout [Fixnum] Number of seconds after which a
+        #   lock obtained by this process will expire. This is to ensure that
+        #   locks don't remain indefinitely if a process fails to release the
+        #   lock properly. As such, it should be longer than any reasonable
+        #   run of this process is likely to take, but no longer.
+        # @!attribute :lock_wait_timeout [Fixnum] Number of seconds before
+        #   this process will give up waiting for a lock to become available.
+        add_properties(:name, :description, :instance, :runs,
+            :lock_name, :lock_timeout, :lock_wait_timeout
+        )
+
+
+        # Create a new instance of this definition.
+        def initialize
+            @runs = []
+        end
+
+
+        # Returns an event name for publication, based on the sub-class of Runnable
+        # that is triggering the event.
+        def event_name(event)
+            "#{self.class.name.split('::')[1].downcase}.#{event}"
+        end
+
+
+        # Sets properties from an options hash.
+        #
+        # @param opts [Hash] A hash containing properties to be set on this
+        #   definable.
+        def set_from_options(opts)
+            unknown = opts.keys - self.class.properties
+            if unknown.size > 0
+                raise ArgumentError, "The following option(s) are invalid for #{
+                    self.class.name}: #{unknown.join(', ')}. Valid options are: #{
+                    self.class.properties.join(', ')}"
+            end
+            self.class.properties.each do |prop|
+                if opts.has_key?(prop)
+                    self.send("#{prop}=", opts[prop])
+                end
+            end
+        end
+
+
+        # Adds an aspect (as in aspect-oriented programming, or AOP) around the
+        # existing instance method +mthd_name+ on +tgt_class+. The aspect does
+        # the following:
+        # - Calls the #pre_execute method with the object instance on which the
+        #   aspect method is being invoked. If the #pre_execute method returns
+        #   false, the method call is skipped; otherwise, proceeds to the next
+        #   step.
+        # - Calls the #around_execute method, which must yield back at the point
+        #   at which the wrapped method body should be invoked.
+        # - Calls the #post_execute method with a boolean OK indicator, and the
+        #   result of the method (if OK) or the exception it threw (if not OK).
+        #
+        # @param tgt_class [Class] The class on which the method to be wrapped
+        #   is defined.
+        # @param mthd_name [Symbol] The name of the instance method to be
+        #   wrapped.
+        def add_aspect(tgt_class, mthd_name)
+            defn = self
+            mthd = tgt_class.instance_method(mthd_name)
+            tgt_class.class_eval do
+                define_method mthd_name do |*args, &block|
+                    run = defn.create_run(self, *args)
+                    if run.pre_execute(self, *args)
+                        ok = false
+                        result = nil
+                        begin
+                            run.around_execute(self, *args) do
+                                result = mthd.bind(self).call(*args, &block)
+                            end
+                            ok = true
+                            run.success(self, result)
+                            result
+                        rescue Exception => ex
+                            run.failure(self, ex) unless ok
+                            raise
+                        rescue Interrupt
+                            run.abort(self) unless ok
+                            raise
+                        ensure
+                            run.post_execute(self, ok)
+                        end
+                    end
+                end
+            end
+            Events.publish(tgt_class, event_name('defined'), mthd_name)
+        end
+
+
+        # Creates an associated Runnable object for this definition. This method
+        # must be overridden in sub-classes.
+        #
+        # @param process_obj [Object] The process object instance on which the
+        #   process method will be invoked.
+        # @param args [Array<Object>] The arguments to be passed to the process
+        #   method.
+        def create_run(process_obj, *args)
+            raise "Not implemented in #{self.class.name}"
+        end
+
+
+        # Add a handler for interrupt (i.e. Ctrl-C etc) signals; this simply
+        # raises an Interrupt exception on the main thread
+        trap 'INT' do
+            Thread.main.raise Interrupt
+        end
+
+    end
+
+end
+

data/lib/batch-kit/framework/job.rb

@@ -0,0 +1,121 @@
+require_relative '../arguments'
+require_relative '../configurable'
+require_relative '../loggable'
+
+
+# Default log level is :detail
+BatchKit::LogManager.configure(log_level: :detail)
+
+
+class BatchKit
+
+    class Job
+
+        include Arguments
+        include Configurable
+        include Loggable
+
+
+        # Include ActsAsJob into any inheriting class
+        def self.inherited(sub_class)
+            sub_class.class_eval do
+                include ActsAsJob
+            end
+        end
+
+
+        # A class variable for controlling whether jobs run; defaults to true.
+        # Provides a means for orchestration programs to prevent the running
+        # of jobs on require when jobs need to be runnable as standalone progs.
+        @@enabled = true
+        def self.enabled=(val)
+            @@enabled = val
+        end
+
+
+        # A method that instantiates an instance of this job, parses
+        # arguments from the command-line, and then executes the job.
+        def self.run(args = ARGV)
+            if @@enabled
+                if args.length == 0 && self.args_def.keys.length > 0
+                    shell
+                else
+                    run_once(args)
+                end
+            end
+        end
+
+
+        # Instantiates and executes a job, using the supplied arguments +args+.
+        #
+        # @param args [Array<String>] an array containin§g the command-line to
+        #   be processed by the job.
+        def self.run_once(args, show_usage_on_error = true)
+            job = self.new
+            job.parse_arguments(args, show_usage_on_error)
+            unless self.job.method_name
+                raise "No job entry method has been defined; use job :<method_name> or job do ... end in your class"
+            end
+            job.send(self.job.method_name)
+        end
+
+
+        # Starts an interactive shell for this job. Each command line entered is
+        # passed to a new instance of the job for execution.
+        def self.shell(prompt = '> ')
+            require 'readline'
+            require 'csv'
+            puts "Starting interactive shell... enter 'exit' to quit"
+            while true do
+                args = Readline.readline(prompt, true)
+                break if args == 'exit' || args == 'quit'
+                begin
+                    run_once(CSV.parse_line(args, col_sep: ' '), false)
+                rescue Exception
+                end
+            end
+        end
+
+
+        # Convenience method for using a lock within a job method
+        #
+        # @param lock_name [String] The name of the lock to obtain during
+        #   execution of the block.
+        # @param lock_timeout [Fixnum] The maximum time (in seconds) until the
+        #   lock should expire.
+        # @param wait_timeout [Fixnum] An optional time (in seconds) to wait for
+        #   the lock to become available if it is already in use.
+        def with_lock(lock_name, lock_timeout, wait_timeout = nil, &blk)
+            self.job_run.with_lock(lock_name, lock_timeout, wait_timeout, &blk)
+        end
+
+
+        Events.subscribe(self, 'job_run.execute') do |obj, run, *args|
+            Console.title = run.label
+        end
+
+        Events.subscribe(self, 'task_run.execute') do |obj, run, *args|
+            Console.title = "#{run.job_run.label} : #{run.label}"
+        end
+
+        Events.subscribe(self, 'task_run.post-execute') do |obj, run, *args|
+            Console.title = run.job_run.label
+        end
+
+
+        # Add unhandled exception logging
+        Events.subscribe(self, ['sequence_run.failure',
+                                       'job_run.failure',
+                                       'task_run.failure']) do |obj, run, ex|
+            unless (oid = ex.object_id) == @last_id
+                @last_id = oid
+                # Strip out framework methods from backtrace
+                ex.backtrace.reject!{ |f| f =~ /lib.batch.framework/ }
+                obj.log.error "#{ex} at #{ex.backtrace.first}"
+            end
+        end
+
+    end
+
+end
+

data/lib/batch-kit/framework/job_definition.rb

@@ -0,0 +1,105 @@
+require 'socket'
+
+
+class BatchKit
+
+    class Job
+
+        # Captures details about a job definition - the class of the job, the server
+        # it runs on, the file it is defined in, etc.
+        class Definition < Definable
+
+            # @!attribute :job_class [Class] The class that defines the job.
+            # @!attribute :method_name [Symbol] The method that is run to execute
+            #   the job.
+            # @!attribute :computer [String] The name of the machine on which the
+            #   job was instantiated.
+            # @!attribute :file [String] The name of the file containing the job
+            #   code.
+            # @!attribute :do_not_track [Boolean] By default, job executions may be
+            #   recorded (if a persistence layer is available). This attribute can be
+            #   used by jobs to indicate that runs of this job should not be recorded.
+            # @!attribute :tasks [Hash<Task::Definition>] A hash of task method names to
+            #   Task::Definition objects capturing details of each task that is defined
+            #   for this Job::Definition.
+            # @!attribute :job_id [Fixnum] A unique id for this Job::Definition, as
+            #   assigned by the persistence layer.
+            # @!attribute :job_version [Fixnum] A version number for the job.
+            add_properties(
+                # Properties from job/task declarations
+                :job_class, :method_name, :computer, :file, :do_not_track, :tasks,
+                # Properties provided by persistence layer
+                :job_id, :job_version
+            )
+
+
+            # Create a new job Definition object for the job defined in +job_class+
+            # in +job_file+.
+            def initialize(job_class, job_file, job_name = nil)
+                raise ArgumentError, "job_class must be a Class" unless job_class.is_a?(Class)
+                @job_class = job_class
+                @file = job_file
+                @name = job_name || job_class.name.gsub(/([^A-Z ])([A-Z])/, '\1 \2').
+                    gsub(/_/, ' ').gsub('::', ':').gsub(/\b([a-z])/) { $1.upcase }
+                @computer = Socket.gethostname
+                @method_name = nil
+                @tasks = {}
+                super()
+            end
+
+
+            # Define a job method - the method to be run to trigger the execution
+            # of the job.
+            #
+            # @param mthd_name [Symbol] The name of a method on the job class
+            #   that is executed to begin the job processing. Note: This method
+            #   must already exist on the job class when this setter is called, so
+            #   that it can be wrapped in an aspect with before/after processing.
+            def method_name=(mthd_name)
+                unless job_class.instance_methods.include?(mthd_name)
+                    raise ArgumentError, "Job class #{job_class.name} does not define a ##{mthd_name} method"
+                end
+                if @method_name
+                    raise "Job class #{job_class.name} already has a job method defined (##{@method_name})"
+                end
+                @method_name = mthd_name
+
+                # Add an aspect for executing job
+                add_aspect(job_class, mthd_name)
+            end
+
+
+            # Add a record of a run of the job, or details about a task that the job
+            # performs.
+            def <<(task)
+                unless task.is_a?(Task::Definition)
+                    raise ArgumentError, "Only a Task::Definition can be added to a Job::Definition"
+                end
+                key = task.method_name
+                if @tasks.has_key?(key)
+                    raise ArgumentError, "#{self} already has a task for ##{key}"
+                end
+                @tasks[key] = task
+            end
+
+
+            # Create a new Job::Run object for a run of thie job.
+            #
+            # @param job_obj [Object] The job object that is running this job.
+            # @param args [Array<Object>] The arguments passed to the job method.
+            def create_run(job_obj, *args)
+                job_run = Job::Run.new(self, job_obj, *args)
+                @runs << job_run
+                job_run
+            end
+
+
+            def to_s
+                "<BatchKit::Job::Definition #{@job_class.name}##{@method_name}>"
+            end
+
+        end
+
+    end
+
+end

data/lib/batch-kit/framework/job_run.rb

@@ -0,0 +1,145 @@
+require 'etc'
+
+
+class BatchKit
+
+    class Job
+
+        # Captures details of an execution of a job.
+        class Run < Runnable
+
+            # @!attribute :run_by [String] The name of the user that ran this job
+            #   instance.
+            # @!attribute :cmd_line [String] The command-line used to invoke the job.
+            # @!attribute :job_args [ArgParser::Arguments] A structure holding the
+            #   parsed job arguments.
+            #
+            # @!attribute :job_run_id [Fixnum] An integer identifier that uniquely
+            #   identifies this job run.
+            # @!attribute :pid [Fixnum] A process identifier (PID) for the process
+            #   that is running the job.
+            # @!attribute :request_id [Fixnum] An integer identifier that links this
+            #   job run to a job run request (if job is run on-demand).
+            # @!attribute :requestors [Array<String>] A list of the requestor(s) that
+            #   requested for this job to be run. May be more than one if the request
+            #   has been in a queue.
+            # @!attribute :start_time [Time] Time at which the job started
+            #   executing.
+            # @!attribute :end_time [Time] Time at which the job ended execution.
+            # @!attribute :task_runs [Array<TaskRun>] An array containing details of
+            #   the tasks that were executed by this job.
+            # @!attribute :exit_code [Fixnum] An exit status code for the job, where
+            #   0 signifies success, and non-zero failure. A value of -1 indicates
+            #   the job was aborted (killed).
+            # @!attribute :exception [Exception] Any uncaught exception that
+            #   occurred during job execution (and which was not caught by a task
+            #   run).
+            PROPERTIES = [
+                :run_by, :pid, :job_run_id,
+                :cmd_line, :job_args,
+                :request_id, :requestors, :task_runs
+            ]
+            # Define accessors for each property
+            PROPERTIES.each do |attr|
+                attr_accessor attr
+            end
+
+            # Make Job::Definition properties accessible off this Job::Run.
+            add_delegated_properties(*Job::Definition.properties)
+
+
+            # Instantiate a new JobRun representing a run of a job.
+            #
+            # @param job_def [Job::Definition] The Job::Definition to which this
+            #   run relates.
+            # @param job_object [Object] The job object instance from which the
+            #   job is being executed.
+            # @param run_args [Array<Object>] An array of the argument values
+            #   passed to the job method.
+            def initialize(job_def, job_object, *run_args)
+                raise ArgumentError unless job_def.is_a?(Job::Definition)
+                @run_by = Etc.getlogin
+                @cmd_line = "#{$0} #{ARGV.map{ |s| s =~ / |^\*$/ ? %Q{"#{s}"} : s }.join(' ')}".strip
+                @pid = ::Process.pid
+                @task_runs = []
+                @job_args = job_object.arguments if job_object.respond_to?(:arguments)
+                super(job_def, job_object, run_args)
+            end
+
+
+            # Adds a Task::Run to this Job::Run.
+            def <<(task_run)
+                unless task_run.is_a?(Task::Run)
+                    raise ArgumentError, "Only Task::Run objects can be added to this Job::Run"
+                end
+                @task_runs << task_run
+            end
+
+
+            # Called as the process is executing.
+            #
+            # @param process_obj [Object] Object that is executing the batch
+            #   process.
+            # @param args [*Object] Any arguments passed to the method that is
+            #   executing the process.
+            # @yield at the point when the process should execute.
+            def around_execute(process_obj, *args)
+              if process_obj.job_run && process_obj.job_run.status == :executing
+                    raise "There is already a job run active (#{process_obj.job_run}) for #{process_obj}"
+                end
+                process_obj.instance_variable_set(:@__job_run__, self)
+                super
+            end
+
+
+            # Called after the process executes and completes successfully.
+            #
+            # @param process_obj [Object] Object that is executing the batch
+            #   process.
+            # @param result [Object] The return value of the process.
+            def success(process_obj, result)
+                super
+                process_obj.on_success if process_obj.respond_to?(:on_success)
+            end
+
+
+            # Called after the process executes and fails.
+            #
+            # @param process_obj [Object] Object that is executing the batch
+            #   process.
+            # @param exception [Exception] The exception that caused the job to
+            #   fail.
+            def failure(process_obj, exception)
+                super
+                process_obj.on_failure(exception) if process_obj.respond_to?(:on_failure)
+            end
+
+
+            # Called if a batch process is aborted.
+            #
+            # @param process_obj [Object] Object that is executing the batch
+            #   process.
+            def abort(process_obj)
+                super
+                process_obj.on_abort if process_obj.respond_to?(:on_abort)
+            end
+
+
+            # @return [Boolean] True if the job run should be recorded via any
+            #   persistence layer.
+            def persist?
+                !definition.do_not_track
+            end
+
+
+            # @return [String] a short representation of this job run.
+            def to_s
+                "<BatchKit::Job::Run label='#{label}'>"
+            end
+
+        end
+
+    end
+
+end
+