batch-kit 0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +22 -0
  3. data/README.md +165 -0
  4. data/lib/batch-kit.rb +9 -0
  5. data/lib/batch-kit/arguments.rb +57 -0
  6. data/lib/batch-kit/config.rb +517 -0
  7. data/lib/batch-kit/configurable.rb +68 -0
  8. data/lib/batch-kit/core_ext/enumerable.rb +97 -0
  9. data/lib/batch-kit/core_ext/file.rb +69 -0
  10. data/lib/batch-kit/core_ext/file_utils.rb +103 -0
  11. data/lib/batch-kit/core_ext/hash.rb +17 -0
  12. data/lib/batch-kit/core_ext/numeric.rb +17 -0
  13. data/lib/batch-kit/core_ext/string.rb +88 -0
  14. data/lib/batch-kit/database.rb +133 -0
  15. data/lib/batch-kit/database/java_util_log_handler.rb +65 -0
  16. data/lib/batch-kit/database/log4r_outputter.rb +57 -0
  17. data/lib/batch-kit/database/models.rb +548 -0
  18. data/lib/batch-kit/database/schema.rb +229 -0
  19. data/lib/batch-kit/encryption.rb +7 -0
  20. data/lib/batch-kit/encryption/java_encryption.rb +178 -0
  21. data/lib/batch-kit/encryption/ruby_encryption.rb +175 -0
  22. data/lib/batch-kit/events.rb +157 -0
  23. data/lib/batch-kit/framework/acts_as_job.rb +197 -0
  24. data/lib/batch-kit/framework/acts_as_sequence.rb +123 -0
  25. data/lib/batch-kit/framework/definable.rb +169 -0
  26. data/lib/batch-kit/framework/job.rb +121 -0
  27. data/lib/batch-kit/framework/job_definition.rb +105 -0
  28. data/lib/batch-kit/framework/job_run.rb +145 -0
  29. data/lib/batch-kit/framework/runnable.rb +235 -0
  30. data/lib/batch-kit/framework/sequence.rb +87 -0
  31. data/lib/batch-kit/framework/sequence_definition.rb +38 -0
  32. data/lib/batch-kit/framework/sequence_run.rb +48 -0
  33. data/lib/batch-kit/framework/task_definition.rb +89 -0
  34. data/lib/batch-kit/framework/task_run.rb +53 -0
  35. data/lib/batch-kit/helpers/date_time.rb +54 -0
  36. data/lib/batch-kit/helpers/email.rb +198 -0
  37. data/lib/batch-kit/helpers/html.rb +175 -0
  38. data/lib/batch-kit/helpers/process.rb +101 -0
  39. data/lib/batch-kit/helpers/zip.rb +30 -0
  40. data/lib/batch-kit/job.rb +11 -0
  41. data/lib/batch-kit/lockable.rb +138 -0
  42. data/lib/batch-kit/loggable.rb +78 -0
  43. data/lib/batch-kit/logging.rb +169 -0
  44. data/lib/batch-kit/logging/java_util_logger.rb +87 -0
  45. data/lib/batch-kit/logging/log4r_logger.rb +71 -0
  46. data/lib/batch-kit/logging/null_logger.rb +35 -0
  47. data/lib/batch-kit/logging/stdout_logger.rb +96 -0
  48. data/lib/batch-kit/resources.rb +191 -0
  49. data/lib/batch-kit/sequence.rb +7 -0
  50. metadata +122 -0
@@ -0,0 +1,235 @@
1
+ require 'forwardable'
2
+
3
+
4
+ class BatchKit
5
+
6
+ # Captures details of a single execution of a runnable batch process, e.g. a
7
+ # Task or Job.
8
+ class Runnable
9
+
10
+ # Runnables delegate to their definitions for properties that are common
11
+ # across all runs.
12
+ extend Forwardable
13
+
14
+ # Add locking functionality for obtaining a lock during execution of a
15
+ # Runnable
16
+ include Lockable
17
+
18
+
19
+ class << self
20
+
21
+ # Add delegates for each specified property in +props+.
22
+ def add_delegated_properties(*props)
23
+ del_props = props.reject{ |prop| self.instance_methods.include?(prop) }
24
+ def_delegators :@definition, *del_props
25
+ end
26
+
27
+ end
28
+
29
+
30
+ # The definition object for this runnable
31
+ attr_reader :definition
32
+ # The object instance that is running this runnable
33
+ attr_reader :object
34
+ # The instance qualifier for this runnable, if it has an instance
35
+ # qualifier.
36
+ attr_reader :instance
37
+ # Current status of this process.
38
+ # One of the following states:
39
+ # :initialized
40
+ # :skipped
41
+ # :executing
42
+ # :completed
43
+ # :failed
44
+ # :aborted
45
+ attr_reader :status
46
+ # Time at which processing began (or nil)
47
+ attr_reader :start_time
48
+ # Time at which processing completed (or nil)
49
+ attr_reader :end_time
50
+ # Exit code of the process
51
+ attr_reader :exit_code
52
+ # Exception thrown that caused process to fail
53
+ attr_accessor :exception
54
+ # Name of any exclusive lock needed by this run
55
+ attr_reader :lock_name
56
+ # Number of seconds before the lock times out
57
+ attr_reader :lock_timeout
58
+ # Number of seconds to wait for the lock to be released before giving up
59
+ attr_reader :lock_wait_timeout
60
+
61
+
62
+
63
+ # Sets the state of the runnable to :initialized.
64
+ def initialize(definition, obj, run_args)
65
+ @definition = definition
66
+ @object = obj
67
+ @instance = eval_property_expr(definition.instance, obj, run_args)
68
+ @status = :initialized
69
+ @lock_name = eval_property_expr(definition.lock_name, obj, run_args)
70
+ @lock_timeout = case definition.lock_timeout
71
+ when Numeric then definition.lock_timeout
72
+ when String then eval_property_expr(definition.lock_timeout, obj, run_args, :to_i)
73
+ end
74
+ @lock_wait_timeout = case definition.lock_wait_timeout
75
+ when Numeric then definition.lock_wait_timeout
76
+ when String then eval_property_expr(definition.lock_wait_timeout, obj, run_args, :to_i)
77
+ end
78
+ Events.publish(self, event_name('initialized'))
79
+ end
80
+
81
+
82
+ # Returns an event name for publication, based on the sub-class of Runnable
83
+ # that is triggering the event.
84
+ def event_name(event)
85
+ "#{self.class.name.split('::')[1..-1].join('_').downcase}.#{event}"
86
+ end
87
+
88
+
89
+ # @return a label consisting of the name and any instance qualifier.
90
+ def label
91
+ lbl = @definition.name.gsub(/_/, ' ').gsub(/\b([a-z])/) { $1.upcase }
92
+ @instance ? "#{lbl} [#{@instance}]" : lbl
93
+ end
94
+
95
+
96
+ # Returns the elapsed time in seconds
97
+ def elapsed
98
+ @start_time ? (@end_time || Time.now) - @start_time : 0
99
+ end
100
+
101
+
102
+ # A pre-execute pointcut for execution of a process. Return value
103
+ # determines whether execution should proceed.
104
+ #
105
+ # @param process_obj [Object] Object that is executing the batch
106
+ # process.
107
+ # @param args [*Object] Any arguments passed to the method that is
108
+ # executing the process.
109
+ # @return [Boolean] True if the process should proceed, or false if it
110
+ # should be skipped.
111
+ def pre_execute(process_obj, *args)
112
+ if Events.has_subscribers?(process_obj, event_name('pre-execute'))
113
+ run = Events.publish(process_obj, event_name('pre-execute'), self, *args)
114
+ else
115
+ run = true
116
+ end
117
+ unless run
118
+ @status = :skipped unless run
119
+ Events.publish(process_obj, event_name('skipped'), self, *args)
120
+ end
121
+ run
122
+ end
123
+
124
+
125
+ # Called as the process is executing.
126
+ #
127
+ # @param process_obj [Object] Object that is executing the batch
128
+ # process.
129
+ # @param args [*Object] Any arguments passed to the method that is
130
+ # executing the process.
131
+ # @yield at the point when the process should execute.
132
+ def around_execute(process_obj, *args, &blk)
133
+ @start_time = Time.now
134
+ @status = :executing
135
+ @exit_code = nil
136
+ Events.publish(process_obj, event_name('execute'), self, *args)
137
+ begin
138
+ if @lock_name
139
+ self.with_lock(@lock_name, @lock_timeout, @lock_wait_timeout, &blk)
140
+ else
141
+ yield
142
+ end
143
+ ensure
144
+ @end_time = Time.now
145
+ end
146
+ end
147
+
148
+
149
+ # Called after the process executes and completes successfully.
150
+ #
151
+ # @param process_obj [Object] Object that is executing the batch
152
+ # process.
153
+ # @param result [Object] The return value of the process.
154
+ def success(process_obj, result)
155
+ @status = :completed
156
+ @exit_code = 0 unless @exit_code
157
+ Events.publish(process_obj, event_name('success'), self, result)
158
+ end
159
+
160
+
161
+ # Called after the process executes and fails.
162
+ #
163
+ # @param process_obj [Object] Object that is executing the batch
164
+ # process.
165
+ # @param exception [Exception] The exception that caused this runnable
166
+ # to fail.
167
+ def failure(process_obj, exception)
168
+ @status = :failed
169
+ @exit_code = 1 unless @exit_code
170
+ @exception = exception
171
+ Events.publish(process_obj, event_name('failure'), self, exception)
172
+ end
173
+
174
+
175
+ # Called if a batch process is aborted.
176
+ #
177
+ # @param process_obj [Object] Object that is executing the batch
178
+ # process.
179
+ def abort(process_obj)
180
+ @status = :aborted
181
+ Events.publish(process_obj, event_name('abort'), self)
182
+ end
183
+
184
+
185
+ # Called after the process executes.
186
+ #
187
+ # @param process_obj [Object] Object that is executing the batch
188
+ # process.
189
+ # @param success [Boolean] True if the process completed without
190
+ # throwing an exception.
191
+ def post_execute(process_obj, success)
192
+ Events.publish(process_obj, event_name('post-execute'), self, success)
193
+ @object = nil
194
+ end
195
+
196
+
197
+ private
198
+
199
+
200
+ # Replaces placeholder expressions in a property expression to return a
201
+ # property value for a job, task, etc. Property expressions may contain
202
+ # both references to arguments passed to a method, as well as Ruby
203
+ # expressions. Both are indicated by %{} or ${} delimiters surrounding
204
+ # the expression to be evaluated and replaced.
205
+ #
206
+ # @param property_expr [String] The expression to be evaluated.
207
+ # @param instance_obj [Object] The object against which Ruby expressions
208
+ # in the property_expr will be evaluated.
209
+ # @param run_args [Array<Object>] An array of arguments passed to the
210
+ # method used to execute the job, task, etc.
211
+ # @param conv_mthd [Symbol] The optional name of a method to call on the
212
+ # result String to convert it to another type (Fixnum, Symbol, etc)
213
+ # @return [Object] The evaluated property value for this run.
214
+ def eval_property_expr(property_expr, instance_obj, run_args, conv_mthd = nil)
215
+ if property_expr
216
+ raise ArgumentError, "property_expr must be a String" unless property_expr.is_a?(String)
217
+ # Replace references to run arguments (i.e. ${0} to ${9}) first...
218
+ property = property_expr.gsub(/(?:\$|%)\{([0-9])\}/) do
219
+ val = run_args[$1.to_i]
220
+ val.is_a?(Array) ? val.join(', ') : val
221
+ end
222
+ # ... then evaluate any remaining expressions between ${} or %{}
223
+ property.gsub!(/(?:\$|%)\{([^\}]+)\}/) do
224
+ val = instance_obj.instance_eval($1)
225
+ val.is_a?(Array) ? val.join(', ') : val
226
+ end
227
+ property = property.length > 0 ?
228
+ (conv_mthd ? property.send(conv_mthd) : property) : nil
229
+ end
230
+ end
231
+
232
+ end
233
+
234
+ end
235
+
@@ -0,0 +1,87 @@
1
+ class BatchKit
2
+
3
+ class Sequence
4
+
5
+ include Arguments
6
+ include Configurable
7
+ include Loggable
8
+
9
+
10
+ # Include ActsAsSequence into any inheriting class
11
+ def self.inherited(sub_class)
12
+ sub_class.class_eval do
13
+ include ActsAsSequence
14
+ end
15
+ end
16
+
17
+
18
+ # A class variable for controlling whether sequences run; defaults to true.
19
+ # Provides a means for orchestration programs to prevent the running
20
+ # of sequences on require when sequences need to be runnable as standalone progs.
21
+ @@enabled = true
22
+ def self.enabled=(val)
23
+ @@enabled = val
24
+ end
25
+
26
+
27
+ # Import arguments defined on a Job into this sequence
28
+ def self.import_args(source, options={})
29
+ unless source.is_a?(ArgParser::Definition)
30
+ source = source.args_def
31
+ end
32
+ exclude = [options[:except]].flatten
33
+ source.args.each do |arg|
34
+ unless exclude.include?(arg.key)
35
+ arg = arg.clone
36
+ if self.args_def.short_keys.include?(arg.short_key)
37
+ arg.instance_variable_set :@short_key, nil
38
+ end
39
+ self.args_def << arg
40
+ end
41
+ end
42
+ end
43
+
44
+
45
+ # A method that instantiates an instance of this job, parses
46
+ # arguments from the command-line, and then executes the job.
47
+ def self.run
48
+ if @@enabled
49
+ sequence = self.new
50
+ sequence.parse_arguments
51
+ unless self.sequence.method_name
52
+ raise "No sequence entry method has been defined; use sequence :<method_name> or sequence do ... end in your class"
53
+ end
54
+ sequence.send(self.sequence.method_name)
55
+ end
56
+ end
57
+
58
+
59
+ def run(job_cls, args)
60
+ job = job_cls.new
61
+ keys, vals = [], []
62
+ job_cls.args_def.args.each do |arg|
63
+ keys << arg.key
64
+ if args.has_key?(arg.key)
65
+ vals << args[arg.key]
66
+ elsif self.args_def.has_key?(arg.key)
67
+ vals << self.arguments.send(arg.key)
68
+ else
69
+ vals << nil
70
+ end
71
+ end
72
+ job_args = Struct.new(*keys)
73
+ job_arg_vals = job_args.new(*vals)
74
+ job.instance_variable_set(:@arguments, job_arg_vals)
75
+ if block_given?
76
+ yield job, job_arg_vals
77
+ else
78
+ unless job_cls.job.method_name
79
+ raise "No job entry method has been defined; use job :<method_name> or job do ... end in your class"
80
+ end
81
+ job.send(job_cls.job.method_name)
82
+ end
83
+ end
84
+
85
+ end
86
+
87
+ end
@@ -0,0 +1,38 @@
1
+ class BatchKit
2
+
3
+ class Sequence
4
+
5
+ # Captures details about a sequence definition: the jobs contained,
6
+ # order of execution, etc.
7
+ class Definition < Definable
8
+
9
+ add_properties(
10
+ # Properties from job/task declarations
11
+ :sequence_class, :method_name, :computer, :file, :do_not_track, :jobs,
12
+ # Properties required by persistence layer
13
+ :sequence_id, :sequence_version
14
+ )
15
+
16
+
17
+ def initialize(sequence_class, sequence_file, sequence_name = nil)
18
+ raise ArgumentError, "sequence_class must be a Class" unless sequence_class.is_a?(Class)
19
+ @sequence_class = sequence_class
20
+ @file = sequence_file
21
+ @name = sequence_name || sequence_class.name.gsub(/([^A-Z ])([A-Z])/, '\1 \2').
22
+ gsub(/_/, ' ').gsub('::', ':').gsub(/\b([a-z])/) { $1.upcase }
23
+ @computer = Socket.gethostname
24
+ @method_name = nil
25
+ @tasks = {}
26
+ super()
27
+ end
28
+
29
+
30
+ def to_s
31
+ "<BatchKit::Sequence::Definition #{name}>"
32
+ end
33
+
34
+ end
35
+
36
+ end
37
+
38
+ end
@@ -0,0 +1,48 @@
1
+ class BatchKit
2
+
3
+ module Sequence
4
+
5
+ # Captures details of an execution of a task.
6
+ class Run < Runnable
7
+
8
+ # @return [Fixnum] An integer identifier that uniquely identifies
9
+ # this task run.
10
+ attr_accessor :sequence_run_id
11
+
12
+ # Make Task::Defintion properties accessible off this Task::Run.
13
+ add_delegated_properties(*Sequence::Definition.properties)
14
+
15
+
16
+ # Create a new sequence run.
17
+ #
18
+ # @param task_def [Sequence::Definition] The Sequence::Definition to
19
+ # which this run relates.
20
+ # @param job_object [Object] The job object instance from which the
21
+ # sequence is being executed.
22
+ # @param run_args [Array<Object>] An array of the argument values
23
+ # passed to the sequence method.
24
+ def initialize(seq_def, job_object, *run_args)
25
+ raise ArgumentError, "seq_def not a Sequence::Definition" unless seq_def.is_a?(Sequence::Definition)
26
+ super(seq_def, job_object, run_args)
27
+ end
28
+
29
+
30
+ # @return [Boolean] True if this sequence run should be persisted in
31
+ # any persistence layer.
32
+ def persist?
33
+ !definition.do_not_track
34
+ end
35
+
36
+
37
+ # @return [String] A short representation of this Sequence::Run.
38
+ def to_s
39
+ "<BatchKit::Sequence::Run label='#{label}'>"
40
+ end
41
+
42
+ end
43
+
44
+ end
45
+
46
+ end
47
+
48
+
@@ -0,0 +1,89 @@
1
+ class BatchKit
2
+
3
+ module Task
4
+
5
+ # Captures details about a task definition - the job that it belongs to, the
6
+ # method name that performs the task work, etc.
7
+ class Definition < Definable
8
+
9
+ # @!attribute :task_name [String] The name of the task (defaults to the
10
+ # method name).
11
+ # @!attribute :job [Job::Definition] The job that this task belongs to.
12
+ # @!attribute :method_name [Symbol] The name of the method that
13
+ # performs the work for this task.
14
+ # @!attribute :task_id [Fixnum] A unique id for this Task::Definition,
15
+ # assigned by the persistence layer.
16
+ add_properties(
17
+ # Properties defined by a task declaration
18
+ :job, :method_name,
19
+ # Properties defined by persistence layer
20
+ :task_id
21
+ )
22
+
23
+
24
+ # Create a new Task::Definition object for the task defined in +job_class+
25
+ # in +method_name+.
26
+ def initialize(job_class, method_name, task_name = nil)
27
+ raise ArgumentError, "job_class must be a Class" unless job_class.is_a?(Class)
28
+ raise ArgumentError, "method_name must be a Symbol" unless method_name.is_a?(Symbol)
29
+ job_defn = job_class.job
30
+ raise ArgumentError, "job_class must have a Job::Definition" unless job_defn
31
+
32
+ @name = task_name || method_name.to_s.gsub(/([^A-Z ])([A-Z])/, '\1 \2').
33
+ gsub(/_/, ' ').gsub('::', ':').gsub(/\b([a-z])/) { $1.upcase }
34
+ @job = job_defn
35
+ @method_name = nil
36
+ self.method_name = method_name
37
+ @job << self
38
+ super()
39
+ end
40
+
41
+
42
+ # Return the class that defines the task.
43
+ def task_class
44
+ @job.job_class
45
+ end
46
+
47
+
48
+ # Define a task method - the method to be run to trigger the execution
49
+ # of a task.
50
+ #
51
+ # @param mthd_name [Symbol] The name of a method on the task class
52
+ # that is executed to begin the task processing. Note: This method
53
+ # must already exist on the task class when this setter is called, so
54
+ # that it can be wrapped in an aspect with before/after processing.
55
+ def method_name=(mthd_name)
56
+ unless task_class.instance_methods.include?(mthd_name)
57
+ raise ArgumentError, "Task class #{task_class.name} does not define a ##{mthd_name} method"
58
+ end
59
+ if @method_name
60
+ raise "Task class #{task_class.name} already has a task method defined for ##{@method_name}"
61
+ end
62
+ @method_name = mthd_name
63
+
64
+ # Add an aspect for executing task
65
+ add_aspect(task_class, mthd_name)
66
+ end
67
+
68
+
69
+ # Create a new Task::Run object for a run of this task.
70
+ #
71
+ # @param job_obj [Object] The job object that is running this task.
72
+ # @param args [Array<Object>] The arguments passed to the task method.
73
+ def create_run(job_obj, *args)
74
+ task_run = Task::Run.new(self, job_obj, job_obj.job_run, *args)
75
+ @runs << task_run
76
+ task_run
77
+ end
78
+
79
+
80
+ def to_s
81
+ "<BatchKit::Task::Definition #{task_class.name}##{@method_name}>"
82
+ end
83
+
84
+ end
85
+
86
+ end
87
+
88
+ end
89
+