batch-kit 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +22 -0
  3. data/README.md +165 -0
  4. data/lib/batch-kit.rb +9 -0
  5. data/lib/batch-kit/arguments.rb +57 -0
  6. data/lib/batch-kit/config.rb +517 -0
  7. data/lib/batch-kit/configurable.rb +68 -0
  8. data/lib/batch-kit/core_ext/enumerable.rb +97 -0
  9. data/lib/batch-kit/core_ext/file.rb +69 -0
  10. data/lib/batch-kit/core_ext/file_utils.rb +103 -0
  11. data/lib/batch-kit/core_ext/hash.rb +17 -0
  12. data/lib/batch-kit/core_ext/numeric.rb +17 -0
  13. data/lib/batch-kit/core_ext/string.rb +88 -0
  14. data/lib/batch-kit/database.rb +133 -0
  15. data/lib/batch-kit/database/java_util_log_handler.rb +65 -0
  16. data/lib/batch-kit/database/log4r_outputter.rb +57 -0
  17. data/lib/batch-kit/database/models.rb +548 -0
  18. data/lib/batch-kit/database/schema.rb +229 -0
  19. data/lib/batch-kit/encryption.rb +7 -0
  20. data/lib/batch-kit/encryption/java_encryption.rb +178 -0
  21. data/lib/batch-kit/encryption/ruby_encryption.rb +175 -0
  22. data/lib/batch-kit/events.rb +157 -0
  23. data/lib/batch-kit/framework/acts_as_job.rb +197 -0
  24. data/lib/batch-kit/framework/acts_as_sequence.rb +123 -0
  25. data/lib/batch-kit/framework/definable.rb +169 -0
  26. data/lib/batch-kit/framework/job.rb +121 -0
  27. data/lib/batch-kit/framework/job_definition.rb +105 -0
  28. data/lib/batch-kit/framework/job_run.rb +145 -0
  29. data/lib/batch-kit/framework/runnable.rb +235 -0
  30. data/lib/batch-kit/framework/sequence.rb +87 -0
  31. data/lib/batch-kit/framework/sequence_definition.rb +38 -0
  32. data/lib/batch-kit/framework/sequence_run.rb +48 -0
  33. data/lib/batch-kit/framework/task_definition.rb +89 -0
  34. data/lib/batch-kit/framework/task_run.rb +53 -0
  35. data/lib/batch-kit/helpers/date_time.rb +54 -0
  36. data/lib/batch-kit/helpers/email.rb +198 -0
  37. data/lib/batch-kit/helpers/html.rb +175 -0
  38. data/lib/batch-kit/helpers/process.rb +101 -0
  39. data/lib/batch-kit/helpers/zip.rb +30 -0
  40. data/lib/batch-kit/job.rb +11 -0
  41. data/lib/batch-kit/lockable.rb +138 -0
  42. data/lib/batch-kit/loggable.rb +78 -0
  43. data/lib/batch-kit/logging.rb +169 -0
  44. data/lib/batch-kit/logging/java_util_logger.rb +87 -0
  45. data/lib/batch-kit/logging/log4r_logger.rb +71 -0
  46. data/lib/batch-kit/logging/null_logger.rb +35 -0
  47. data/lib/batch-kit/logging/stdout_logger.rb +96 -0
  48. data/lib/batch-kit/resources.rb +191 -0
  49. data/lib/batch-kit/sequence.rb +7 -0
  50. metadata +122 -0
@@ -0,0 +1,169 @@
1
+ class BatchKit
2
+
3
+ # Captures details of a definable batch process, e.g. a Task or Job.
4
+ #
5
+ # @abstract
6
+ class Definable
7
+
8
+ class << self
9
+
10
+ # Register additional properties to be recorded on this definable.
11
+ #
12
+ # @param props [Array<Symbol>] The names of properties to be added
13
+ # to the definition. Used by sub-classes to add to the available
14
+ # properties. This provides a mechanism by which associated
15
+ # Run objects can obtain a list of process properties that they
16
+ # can delegate.
17
+ def add_properties(*props)
18
+ attr_accessor(*props)
19
+ properties.concat(props)
20
+ end
21
+
22
+
23
+ # @return [Array<Symbol>] the names of properties available on this
24
+ # definition.
25
+ def properties
26
+ @properties ||= []
27
+ end
28
+
29
+
30
+ # When this class is inherited, we need to copy the common property
31
+ # names into the sub-class, since each sub-class needs the common
32
+ # property names defined on this base class, as well as any sub-
33
+ # class specific properties.
34
+ def inherited(subclass)
35
+ subclass.instance_variable_set(:@properties, @properties.clone)
36
+ end
37
+
38
+ end
39
+
40
+
41
+ # @!attribute :name [String] A user-friendly name for this process.
42
+ # @!attribute :description [String] A short description of what this
43
+ # process does.
44
+ # @!attribute :instance [String] An optional expression used to assign
45
+ # an instance identifier to a process.
46
+ # An instance identifier allows a process that has different execution
47
+ # profiles (typically depending on the arguments it is run with)
48
+ # to identify which of those profiles it is executing. This expression
49
+ # will be evaluated at the time this process is invoked, and the
50
+ # result will become the instance identifier for the Runnable.
51
+ # @!attribute :runs [Array<Runnable>] Array of runs of this process.
52
+ # @!attribute :lock_name [String] The name of any lock that this
53
+ # process requires before it can proceed. If nil, no lock is
54
+ # required and the process can commence without any co-ordination
55
+ # with other processes.
56
+ # @!attribute :lock_timeout [Fixnum] Number of seconds after which a
57
+ # lock obtained by this process will expire. This is to ensure that
58
+ # locks don't remain indefinitely if a process fails to release the
59
+ # lock properly. As such, it should be longer than any reasonable
60
+ # run of this process is likely to take, but no longer.
61
+ # @!attribute :lock_wait_timeout [Fixnum] Number of seconds before
62
+ # this process will give up waiting for a lock to become available.
63
+ add_properties(:name, :description, :instance, :runs,
64
+ :lock_name, :lock_timeout, :lock_wait_timeout
65
+ )
66
+
67
+
68
+ # Create a new instance of this definition.
69
+ def initialize
70
+ @runs = []
71
+ end
72
+
73
+
74
+ # Returns an event name for publication, based on the sub-class of Runnable
75
+ # that is triggering the event.
76
+ def event_name(event)
77
+ "#{self.class.name.split('::')[1].downcase}.#{event}"
78
+ end
79
+
80
+
81
+ # Sets properties from an options hash.
82
+ #
83
+ # @param opts [Hash] A hash containing properties to be set on this
84
+ # definable.
85
+ def set_from_options(opts)
86
+ unknown = opts.keys - self.class.properties
87
+ if unknown.size > 0
88
+ raise ArgumentError, "The following option(s) are invalid for #{
89
+ self.class.name}: #{unknown.join(', ')}. Valid options are: #{
90
+ self.class.properties.join(', ')}"
91
+ end
92
+ self.class.properties.each do |prop|
93
+ if opts.has_key?(prop)
94
+ self.send("#{prop}=", opts[prop])
95
+ end
96
+ end
97
+ end
98
+
99
+
100
+ # Adds an aspect (as in aspect-oriented programming, or AOP) around the
101
+ # existing instance method +mthd_name+ on +tgt_class+. The aspect does
102
+ # the following:
103
+ # - Calls the #pre_execute method with the object instance on which the
104
+ # aspect method is being invoked. If the #pre_execute method returns
105
+ # false, the method call is skipped; otherwise, proceeds to the next
106
+ # step.
107
+ # - Calls the #around_execute method, which must yield back at the point
108
+ # at which the wrapped method body should be invoked.
109
+ # - Calls the #post_execute method with a boolean OK indicator, and the
110
+ # result of the method (if OK) or the exception it threw (if not OK).
111
+ #
112
+ # @param tgt_class [Class] The class on which the method to be wrapped
113
+ # is defined.
114
+ # @param mthd_name [Symbol] The name of the instance method to be
115
+ # wrapped.
116
+ def add_aspect(tgt_class, mthd_name)
117
+ defn = self
118
+ mthd = tgt_class.instance_method(mthd_name)
119
+ tgt_class.class_eval do
120
+ define_method mthd_name do |*args, &block|
121
+ run = defn.create_run(self, *args)
122
+ if run.pre_execute(self, *args)
123
+ ok = false
124
+ result = nil
125
+ begin
126
+ run.around_execute(self, *args) do
127
+ result = mthd.bind(self).call(*args, &block)
128
+ end
129
+ ok = true
130
+ run.success(self, result)
131
+ result
132
+ rescue Exception => ex
133
+ run.failure(self, ex) unless ok
134
+ raise
135
+ rescue Interrupt
136
+ run.abort(self) unless ok
137
+ raise
138
+ ensure
139
+ run.post_execute(self, ok)
140
+ end
141
+ end
142
+ end
143
+ end
144
+ Events.publish(tgt_class, event_name('defined'), mthd_name)
145
+ end
146
+
147
+
148
+ # Creates an associated Runnable object for this definition. This method
149
+ # must be overridden in sub-classes.
150
+ #
151
+ # @param process_obj [Object] The process object instance on which the
152
+ # process method will be invoked.
153
+ # @param args [Array<Object>] The arguments to be passed to the process
154
+ # method.
155
+ def create_run(process_obj, *args)
156
+ raise "Not implemented in #{self.class.name}"
157
+ end
158
+
159
+
160
+ # Add a handler for interrupt (i.e. Ctrl-C etc) signals; this simply
161
+ # raises an Interrupt exception on the main thread
162
+ trap 'INT' do
163
+ Thread.main.raise Interrupt
164
+ end
165
+
166
+ end
167
+
168
+ end
169
+
@@ -0,0 +1,121 @@
1
+ require_relative '../arguments'
2
+ require_relative '../configurable'
3
+ require_relative '../loggable'
4
+
5
+
6
+ # Default log level is :detail
7
+ BatchKit::LogManager.configure(log_level: :detail)
8
+
9
+
10
+ class BatchKit
11
+
12
+ class Job
13
+
14
+ include Arguments
15
+ include Configurable
16
+ include Loggable
17
+
18
+
19
+ # Include ActsAsJob into any inheriting class
20
+ def self.inherited(sub_class)
21
+ sub_class.class_eval do
22
+ include ActsAsJob
23
+ end
24
+ end
25
+
26
+
27
+ # A class variable for controlling whether jobs run; defaults to true.
28
+ # Provides a means for orchestration programs to prevent the running
29
+ # of jobs on require when jobs need to be runnable as standalone progs.
30
+ @@enabled = true
31
+ def self.enabled=(val)
32
+ @@enabled = val
33
+ end
34
+
35
+
36
+ # A method that instantiates an instance of this job, parses
37
+ # arguments from the command-line, and then executes the job.
38
+ def self.run(args = ARGV)
39
+ if @@enabled
40
+ if args.length == 0 && self.args_def.keys.length > 0
41
+ shell
42
+ else
43
+ run_once(args)
44
+ end
45
+ end
46
+ end
47
+
48
+
49
+ # Instantiates and executes a job, using the supplied arguments +args+.
50
+ #
51
+ # @param args [Array<String>] an array containin§g the command-line to
52
+ # be processed by the job.
53
+ def self.run_once(args, show_usage_on_error = true)
54
+ job = self.new
55
+ job.parse_arguments(args, show_usage_on_error)
56
+ unless self.job.method_name
57
+ raise "No job entry method has been defined; use job :<method_name> or job do ... end in your class"
58
+ end
59
+ job.send(self.job.method_name)
60
+ end
61
+
62
+
63
+ # Starts an interactive shell for this job. Each command line entered is
64
+ # passed to a new instance of the job for execution.
65
+ def self.shell(prompt = '> ')
66
+ require 'readline'
67
+ require 'csv'
68
+ puts "Starting interactive shell... enter 'exit' to quit"
69
+ while true do
70
+ args = Readline.readline(prompt, true)
71
+ break if args == 'exit' || args == 'quit'
72
+ begin
73
+ run_once(CSV.parse_line(args, col_sep: ' '), false)
74
+ rescue Exception
75
+ end
76
+ end
77
+ end
78
+
79
+
80
+ # Convenience method for using a lock within a job method
81
+ #
82
+ # @param lock_name [String] The name of the lock to obtain during
83
+ # execution of the block.
84
+ # @param lock_timeout [Fixnum] The maximum time (in seconds) until the
85
+ # lock should expire.
86
+ # @param wait_timeout [Fixnum] An optional time (in seconds) to wait for
87
+ # the lock to become available if it is already in use.
88
+ def with_lock(lock_name, lock_timeout, wait_timeout = nil, &blk)
89
+ self.job_run.with_lock(lock_name, lock_timeout, wait_timeout, &blk)
90
+ end
91
+
92
+
93
+ Events.subscribe(self, 'job_run.execute') do |obj, run, *args|
94
+ Console.title = run.label
95
+ end
96
+
97
+ Events.subscribe(self, 'task_run.execute') do |obj, run, *args|
98
+ Console.title = "#{run.job_run.label} : #{run.label}"
99
+ end
100
+
101
+ Events.subscribe(self, 'task_run.post-execute') do |obj, run, *args|
102
+ Console.title = run.job_run.label
103
+ end
104
+
105
+
106
+ # Add unhandled exception logging
107
+ Events.subscribe(self, ['sequence_run.failure',
108
+ 'job_run.failure',
109
+ 'task_run.failure']) do |obj, run, ex|
110
+ unless (oid = ex.object_id) == @last_id
111
+ @last_id = oid
112
+ # Strip out framework methods from backtrace
113
+ ex.backtrace.reject!{ |f| f =~ /lib.batch.framework/ }
114
+ obj.log.error "#{ex} at #{ex.backtrace.first}"
115
+ end
116
+ end
117
+
118
+ end
119
+
120
+ end
121
+
@@ -0,0 +1,105 @@
1
+ require 'socket'
2
+
3
+
4
+ class BatchKit
5
+
6
+ class Job
7
+
8
+ # Captures details about a job definition - the class of the job, the server
9
+ # it runs on, the file it is defined in, etc.
10
+ class Definition < Definable
11
+
12
+ # @!attribute :job_class [Class] The class that defines the job.
13
+ # @!attribute :method_name [Symbol] The method that is run to execute
14
+ # the job.
15
+ # @!attribute :computer [String] The name of the machine on which the
16
+ # job was instantiated.
17
+ # @!attribute :file [String] The name of the file containing the job
18
+ # code.
19
+ # @!attribute :do_not_track [Boolean] By default, job executions may be
20
+ # recorded (if a persistence layer is available). This attribute can be
21
+ # used by jobs to indicate that runs of this job should not be recorded.
22
+ # @!attribute :tasks [Hash<Task::Definition>] A hash of task method names to
23
+ # Task::Definition objects capturing details of each task that is defined
24
+ # for this Job::Definition.
25
+ # @!attribute :job_id [Fixnum] A unique id for this Job::Definition, as
26
+ # assigned by the persistence layer.
27
+ # @!attribute :job_version [Fixnum] A version number for the job.
28
+ add_properties(
29
+ # Properties from job/task declarations
30
+ :job_class, :method_name, :computer, :file, :do_not_track, :tasks,
31
+ # Properties provided by persistence layer
32
+ :job_id, :job_version
33
+ )
34
+
35
+
36
+ # Create a new job Definition object for the job defined in +job_class+
37
+ # in +job_file+.
38
+ def initialize(job_class, job_file, job_name = nil)
39
+ raise ArgumentError, "job_class must be a Class" unless job_class.is_a?(Class)
40
+ @job_class = job_class
41
+ @file = job_file
42
+ @name = job_name || job_class.name.gsub(/([^A-Z ])([A-Z])/, '\1 \2').
43
+ gsub(/_/, ' ').gsub('::', ':').gsub(/\b([a-z])/) { $1.upcase }
44
+ @computer = Socket.gethostname
45
+ @method_name = nil
46
+ @tasks = {}
47
+ super()
48
+ end
49
+
50
+
51
+ # Define a job method - the method to be run to trigger the execution
52
+ # of the job.
53
+ #
54
+ # @param mthd_name [Symbol] The name of a method on the job class
55
+ # that is executed to begin the job processing. Note: This method
56
+ # must already exist on the job class when this setter is called, so
57
+ # that it can be wrapped in an aspect with before/after processing.
58
+ def method_name=(mthd_name)
59
+ unless job_class.instance_methods.include?(mthd_name)
60
+ raise ArgumentError, "Job class #{job_class.name} does not define a ##{mthd_name} method"
61
+ end
62
+ if @method_name
63
+ raise "Job class #{job_class.name} already has a job method defined (##{@method_name})"
64
+ end
65
+ @method_name = mthd_name
66
+
67
+ # Add an aspect for executing job
68
+ add_aspect(job_class, mthd_name)
69
+ end
70
+
71
+
72
+ # Add a record of a run of the job, or details about a task that the job
73
+ # performs.
74
+ def <<(task)
75
+ unless task.is_a?(Task::Definition)
76
+ raise ArgumentError, "Only a Task::Definition can be added to a Job::Definition"
77
+ end
78
+ key = task.method_name
79
+ if @tasks.has_key?(key)
80
+ raise ArgumentError, "#{self} already has a task for ##{key}"
81
+ end
82
+ @tasks[key] = task
83
+ end
84
+
85
+
86
+ # Create a new Job::Run object for a run of thie job.
87
+ #
88
+ # @param job_obj [Object] The job object that is running this job.
89
+ # @param args [Array<Object>] The arguments passed to the job method.
90
+ def create_run(job_obj, *args)
91
+ job_run = Job::Run.new(self, job_obj, *args)
92
+ @runs << job_run
93
+ job_run
94
+ end
95
+
96
+
97
+ def to_s
98
+ "<BatchKit::Job::Definition #{@job_class.name}##{@method_name}>"
99
+ end
100
+
101
+ end
102
+
103
+ end
104
+
105
+ end
@@ -0,0 +1,145 @@
1
+ require 'etc'
2
+
3
+
4
+ class BatchKit
5
+
6
+ class Job
7
+
8
+ # Captures details of an execution of a job.
9
+ class Run < Runnable
10
+
11
+ # @!attribute :run_by [String] The name of the user that ran this job
12
+ # instance.
13
+ # @!attribute :cmd_line [String] The command-line used to invoke the job.
14
+ # @!attribute :job_args [ArgParser::Arguments] A structure holding the
15
+ # parsed job arguments.
16
+ #
17
+ # @!attribute :job_run_id [Fixnum] An integer identifier that uniquely
18
+ # identifies this job run.
19
+ # @!attribute :pid [Fixnum] A process identifier (PID) for the process
20
+ # that is running the job.
21
+ # @!attribute :request_id [Fixnum] An integer identifier that links this
22
+ # job run to a job run request (if job is run on-demand).
23
+ # @!attribute :requestors [Array<String>] A list of the requestor(s) that
24
+ # requested for this job to be run. May be more than one if the request
25
+ # has been in a queue.
26
+ # @!attribute :start_time [Time] Time at which the job started
27
+ # executing.
28
+ # @!attribute :end_time [Time] Time at which the job ended execution.
29
+ # @!attribute :task_runs [Array<TaskRun>] An array containing details of
30
+ # the tasks that were executed by this job.
31
+ # @!attribute :exit_code [Fixnum] An exit status code for the job, where
32
+ # 0 signifies success, and non-zero failure. A value of -1 indicates
33
+ # the job was aborted (killed).
34
+ # @!attribute :exception [Exception] Any uncaught exception that
35
+ # occurred during job execution (and which was not caught by a task
36
+ # run).
37
+ PROPERTIES = [
38
+ :run_by, :pid, :job_run_id,
39
+ :cmd_line, :job_args,
40
+ :request_id, :requestors, :task_runs
41
+ ]
42
+ # Define accessors for each property
43
+ PROPERTIES.each do |attr|
44
+ attr_accessor attr
45
+ end
46
+
47
+ # Make Job::Definition properties accessible off this Job::Run.
48
+ add_delegated_properties(*Job::Definition.properties)
49
+
50
+
51
+ # Instantiate a new JobRun representing a run of a job.
52
+ #
53
+ # @param job_def [Job::Definition] The Job::Definition to which this
54
+ # run relates.
55
+ # @param job_object [Object] The job object instance from which the
56
+ # job is being executed.
57
+ # @param run_args [Array<Object>] An array of the argument values
58
+ # passed to the job method.
59
+ def initialize(job_def, job_object, *run_args)
60
+ raise ArgumentError unless job_def.is_a?(Job::Definition)
61
+ @run_by = Etc.getlogin
62
+ @cmd_line = "#{$0} #{ARGV.map{ |s| s =~ / |^\*$/ ? %Q{"#{s}"} : s }.join(' ')}".strip
63
+ @pid = ::Process.pid
64
+ @task_runs = []
65
+ @job_args = job_object.arguments if job_object.respond_to?(:arguments)
66
+ super(job_def, job_object, run_args)
67
+ end
68
+
69
+
70
+ # Adds a Task::Run to this Job::Run.
71
+ def <<(task_run)
72
+ unless task_run.is_a?(Task::Run)
73
+ raise ArgumentError, "Only Task::Run objects can be added to this Job::Run"
74
+ end
75
+ @task_runs << task_run
76
+ end
77
+
78
+
79
+ # Called as the process is executing.
80
+ #
81
+ # @param process_obj [Object] Object that is executing the batch
82
+ # process.
83
+ # @param args [*Object] Any arguments passed to the method that is
84
+ # executing the process.
85
+ # @yield at the point when the process should execute.
86
+ def around_execute(process_obj, *args)
87
+ if process_obj.job_run && process_obj.job_run.status == :executing
88
+ raise "There is already a job run active (#{process_obj.job_run}) for #{process_obj}"
89
+ end
90
+ process_obj.instance_variable_set(:@__job_run__, self)
91
+ super
92
+ end
93
+
94
+
95
+ # Called after the process executes and completes successfully.
96
+ #
97
+ # @param process_obj [Object] Object that is executing the batch
98
+ # process.
99
+ # @param result [Object] The return value of the process.
100
+ def success(process_obj, result)
101
+ super
102
+ process_obj.on_success if process_obj.respond_to?(:on_success)
103
+ end
104
+
105
+
106
+ # Called after the process executes and fails.
107
+ #
108
+ # @param process_obj [Object] Object that is executing the batch
109
+ # process.
110
+ # @param exception [Exception] The exception that caused the job to
111
+ # fail.
112
+ def failure(process_obj, exception)
113
+ super
114
+ process_obj.on_failure(exception) if process_obj.respond_to?(:on_failure)
115
+ end
116
+
117
+
118
+ # Called if a batch process is aborted.
119
+ #
120
+ # @param process_obj [Object] Object that is executing the batch
121
+ # process.
122
+ def abort(process_obj)
123
+ super
124
+ process_obj.on_abort if process_obj.respond_to?(:on_abort)
125
+ end
126
+
127
+
128
+ # @return [Boolean] True if the job run should be recorded via any
129
+ # persistence layer.
130
+ def persist?
131
+ !definition.do_not_track
132
+ end
133
+
134
+
135
+ # @return [String] a short representation of this job run.
136
+ def to_s
137
+ "<BatchKit::Job::Run label='#{label}'>"
138
+ end
139
+
140
+ end
141
+
142
+ end
143
+
144
+ end
145
+