batch-kit 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +22 -0
- data/README.md +165 -0
- data/lib/batch-kit.rb +9 -0
- data/lib/batch-kit/arguments.rb +57 -0
- data/lib/batch-kit/config.rb +517 -0
- data/lib/batch-kit/configurable.rb +68 -0
- data/lib/batch-kit/core_ext/enumerable.rb +97 -0
- data/lib/batch-kit/core_ext/file.rb +69 -0
- data/lib/batch-kit/core_ext/file_utils.rb +103 -0
- data/lib/batch-kit/core_ext/hash.rb +17 -0
- data/lib/batch-kit/core_ext/numeric.rb +17 -0
- data/lib/batch-kit/core_ext/string.rb +88 -0
- data/lib/batch-kit/database.rb +133 -0
- data/lib/batch-kit/database/java_util_log_handler.rb +65 -0
- data/lib/batch-kit/database/log4r_outputter.rb +57 -0
- data/lib/batch-kit/database/models.rb +548 -0
- data/lib/batch-kit/database/schema.rb +229 -0
- data/lib/batch-kit/encryption.rb +7 -0
- data/lib/batch-kit/encryption/java_encryption.rb +178 -0
- data/lib/batch-kit/encryption/ruby_encryption.rb +175 -0
- data/lib/batch-kit/events.rb +157 -0
- data/lib/batch-kit/framework/acts_as_job.rb +197 -0
- data/lib/batch-kit/framework/acts_as_sequence.rb +123 -0
- data/lib/batch-kit/framework/definable.rb +169 -0
- data/lib/batch-kit/framework/job.rb +121 -0
- data/lib/batch-kit/framework/job_definition.rb +105 -0
- data/lib/batch-kit/framework/job_run.rb +145 -0
- data/lib/batch-kit/framework/runnable.rb +235 -0
- data/lib/batch-kit/framework/sequence.rb +87 -0
- data/lib/batch-kit/framework/sequence_definition.rb +38 -0
- data/lib/batch-kit/framework/sequence_run.rb +48 -0
- data/lib/batch-kit/framework/task_definition.rb +89 -0
- data/lib/batch-kit/framework/task_run.rb +53 -0
- data/lib/batch-kit/helpers/date_time.rb +54 -0
- data/lib/batch-kit/helpers/email.rb +198 -0
- data/lib/batch-kit/helpers/html.rb +175 -0
- data/lib/batch-kit/helpers/process.rb +101 -0
- data/lib/batch-kit/helpers/zip.rb +30 -0
- data/lib/batch-kit/job.rb +11 -0
- data/lib/batch-kit/lockable.rb +138 -0
- data/lib/batch-kit/loggable.rb +78 -0
- data/lib/batch-kit/logging.rb +169 -0
- data/lib/batch-kit/logging/java_util_logger.rb +87 -0
- data/lib/batch-kit/logging/log4r_logger.rb +71 -0
- data/lib/batch-kit/logging/null_logger.rb +35 -0
- data/lib/batch-kit/logging/stdout_logger.rb +96 -0
- data/lib/batch-kit/resources.rb +191 -0
- data/lib/batch-kit/sequence.rb +7 -0
- metadata +122 -0
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'java'
|
2
|
+
|
3
|
+
|
4
|
+
class BatchKit
|
5
|
+
|
6
|
+
module Logging
|
7
|
+
|
8
|
+
class JavaLogFacade
|
9
|
+
|
10
|
+
LEVEL_MAP = {
|
11
|
+
:error => Java::JavaUtilLogging::Level::SEVERE,
|
12
|
+
:warning => Java::JavaUtilLogging::Level::WARNING,
|
13
|
+
:info => Java::JavaUtilLogging::Level::INFO,
|
14
|
+
:config => Java::JavaUtilLogging::Level::CONFIG,
|
15
|
+
:detail => Java::JavaUtilLogging::Level::FINE,
|
16
|
+
:trace => Java::JavaUtilLogging::Level::FINER,
|
17
|
+
:debug => Java::JavaUtilLogging::Level::FINEST
|
18
|
+
}
|
19
|
+
|
20
|
+
# @return The path to any log file used with this logger
|
21
|
+
attr_reader :log_file
|
22
|
+
|
23
|
+
|
24
|
+
def initialize(logger)
|
25
|
+
@java_logger = logger
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
def level
|
30
|
+
LEVEL_MAP.invert[@java_logger.getLevel()]
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
def level=(level)
|
35
|
+
@java_logger.setLevel(LEVEL_MAP[level])
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
# Adds a FileHandler to capture output from this logger to a log file.
|
40
|
+
def log_file=(log_path)
|
41
|
+
@java_logger.getHandlers().each do |h|
|
42
|
+
if h.is_a?(Java::JavaUtilLogging::FileHandler)
|
43
|
+
@java_logger.removeHandler(h)
|
44
|
+
h.close()
|
45
|
+
end
|
46
|
+
end
|
47
|
+
@log_file = log_path
|
48
|
+
if log_path
|
49
|
+
# Java logger does not follow changes in working directory via Dir.chdir
|
50
|
+
log_path = File.absolute_path(log_path)
|
51
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
52
|
+
fh = Java::JavaUtilLogging::FileHandler.new(log_path, true)
|
53
|
+
if defined?(Console::JavaUtilLogger)
|
54
|
+
fmt = Console::JavaUtilLogger::RubyFormatter.new('[%1$tF %1$tT] %4$-6s %5$s', -1)
|
55
|
+
fmt.level_labels[Java::JavaUtilLogging::Level::FINE] = 'DETAIL'
|
56
|
+
fmt.level_labels[Java::JavaUtilLogging::Level::FINER] = 'TRACE'
|
57
|
+
else
|
58
|
+
fmt = Java::JavaUtilLogging::SimpleFormatter.new
|
59
|
+
end
|
60
|
+
fh.setFormatter(fmt)
|
61
|
+
self.addHandler(fh)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
BatchKit::Logging::LEVELS.each do |lvl|
|
67
|
+
java_mthd = LEVEL_MAP[lvl].getName().downcase.intern
|
68
|
+
class_eval <<-EOD
|
69
|
+
def #{lvl}(msg)
|
70
|
+
@java_logger.#{java_mthd}(msg.to_s)
|
71
|
+
end
|
72
|
+
EOD
|
73
|
+
end
|
74
|
+
|
75
|
+
alias_method :warn, :warning
|
76
|
+
|
77
|
+
|
78
|
+
def method_missing(mthd, *args)
|
79
|
+
@java_logger.send(mthd, *args)
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'log4r'
|
2
|
+
require 'log4r/configurator'
|
3
|
+
|
4
|
+
Log4r::Configurator.custom_levels(*BatchKit::Logging::LEVELS.reverse.map{ |l| l.to_s.upcase })
|
5
|
+
|
6
|
+
|
7
|
+
|
8
|
+
class BatchKit
|
9
|
+
|
10
|
+
module Logging
|
11
|
+
|
12
|
+
class Log4rFacade
|
13
|
+
|
14
|
+
def initialize(logger)
|
15
|
+
@log4r_logger = logger
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
def level
|
20
|
+
Log4r::LNAMES[@log4r_logger.level].downcase.intern
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
def level=(lvl)
|
25
|
+
@log4r_logger.level = Log4r::LNAMES.index(lvl.to_s.upcase)
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
def log_file
|
30
|
+
out_name = "#{self.name}_file"
|
31
|
+
fo = @log4r_logger.outputters.find{ |o| o.name == out_name }
|
32
|
+
fo && fo.filename
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
def log_file=(log_path)
|
37
|
+
out_name = "#{self.name}_file"
|
38
|
+
if outputter = Log4r::Outputter[out_name]
|
39
|
+
outputter.close
|
40
|
+
@log4r_logger.remove out_name
|
41
|
+
end
|
42
|
+
if log_path
|
43
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
44
|
+
formatter = Log4r::PatternFormatter.new(pattern: '[%d] %-6l %x %M\r')
|
45
|
+
outputter = Log4r::FileOutputter.new(out_name, filename: log_path,
|
46
|
+
trunc: false, formatter: formatter)
|
47
|
+
@log4r_logger.add out_name
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
BatchKit::Logging::LEVELS.each do |lvl|
|
53
|
+
class_eval <<-EOD
|
54
|
+
def #{lvl}(msg)
|
55
|
+
@log4r_logger.#{lvl}(msg)
|
56
|
+
end
|
57
|
+
EOD
|
58
|
+
end
|
59
|
+
|
60
|
+
alias_method :warn, :warning
|
61
|
+
|
62
|
+
|
63
|
+
def method_missing(mthd, *args)
|
64
|
+
@log4r_logger.send(mthd, *args)
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
class BatchKit
|
2
|
+
|
3
|
+
module Logging
|
4
|
+
|
5
|
+
# Implements a NULL logger, i.e. a logger that throws away log messages.
|
6
|
+
# This logger should be used when no logging is desired.
|
7
|
+
class NullLogger
|
8
|
+
|
9
|
+
|
10
|
+
def self.instance
|
11
|
+
@instance ||= self.new
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def log_level=(level)
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
LEVELS.each do |level|
|
20
|
+
define_method level do |*args|
|
21
|
+
log_msg(level, *args)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
alias_method :warn, :warning
|
26
|
+
|
27
|
+
|
28
|
+
def log_msg(level, msg)
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
class BatchKit
|
2
|
+
|
3
|
+
module Logging
|
4
|
+
|
5
|
+
class StdOutLogger
|
6
|
+
|
7
|
+
def self.logger(name)
|
8
|
+
@loggers ||= {}
|
9
|
+
@loggers[name] ||= self.new(name)
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
# @return [String] The name of this logger
|
14
|
+
attr_reader :name
|
15
|
+
# @return [Symbol] The current level at which logging is set
|
16
|
+
attr_accessor :level
|
17
|
+
# @return [String] The log file path, if any
|
18
|
+
attr_reader :log_file
|
19
|
+
|
20
|
+
# Width at which to split lines
|
21
|
+
attr_accessor :width
|
22
|
+
# Amount by which to indent lines
|
23
|
+
attr_accessor :indent
|
24
|
+
|
25
|
+
|
26
|
+
def initialize(name, level = :detail)
|
27
|
+
@name = name
|
28
|
+
@level = level
|
29
|
+
@indent = 8
|
30
|
+
@width = Console.width if use_console?
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
LEVELS.each do |level|
|
35
|
+
define_method level do |*args|
|
36
|
+
log_msg(level, *args)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
alias_method :warn, :warning
|
41
|
+
|
42
|
+
|
43
|
+
def log_file=(log_path, options = {})
|
44
|
+
@log_file.close if @log_file
|
45
|
+
if log_path
|
46
|
+
append = options.fetch(:append, true)
|
47
|
+
@log_file = File.new(log_path, append ? 'a' : 'w')
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
def log_msg(level, *args)
|
53
|
+
return if LEVELS.index(level) > LEVELS.index(@level)
|
54
|
+
lvl = level.to_s.upcase
|
55
|
+
msg = args.join(' ')
|
56
|
+
spacer = LEVELS.index(level) >= LEVELS.index(:config) ? ' ' : ''
|
57
|
+
fmt_msg = "%-6s %s%s" % [lvl, spacer, msg]
|
58
|
+
if use_console?
|
59
|
+
color = case level
|
60
|
+
when :error then :red
|
61
|
+
when :warning then :yellow
|
62
|
+
when :info then :white
|
63
|
+
when :config then :cyan
|
64
|
+
when :detail then :light_gray
|
65
|
+
else :dark_gray
|
66
|
+
end
|
67
|
+
|
68
|
+
indent = @indent || 0
|
69
|
+
indent += 2 if indent > 0 && [:config, :detail, :trace, :debug].include?(level)
|
70
|
+
|
71
|
+
msg = @width ? Console.wrap_text(msg, @width - indent) : [msg]
|
72
|
+
msg = msg.each_with_index.map do |line, i|
|
73
|
+
"%-6s %s%s" % [[lvl][i], spacer, line]
|
74
|
+
end.join("\n")
|
75
|
+
Console.puts msg, color
|
76
|
+
else
|
77
|
+
STDOUT.puts fmt_msg
|
78
|
+
end
|
79
|
+
if @log_file
|
80
|
+
@log_file.puts Time.now.strftime('[%F %T] ') + fmt_msg
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
def use_console?
|
86
|
+
unless @use_console
|
87
|
+
@use_console = defined?(::Console)
|
88
|
+
end
|
89
|
+
@use_console
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,191 @@
|
|
1
|
+
require 'set'
|
2
|
+
require_relative 'events'
|
3
|
+
|
4
|
+
|
5
|
+
class BatchKit
|
6
|
+
|
7
|
+
# Defines a manager for resource types, such as database connections etc.
|
8
|
+
# Resource types are registered with this class, which then adds acquisition
|
9
|
+
# methods to the ResourceHelper module. These acquisition methods add the
|
10
|
+
# acquired objects to a collection managed by the objects of the class that
|
11
|
+
# includes the ResourceHelper, and modify the returned resource objects so
|
12
|
+
# that they automatically de-register themselves if they are disposed of
|
13
|
+
# explicitly.
|
14
|
+
class ResourceManager
|
15
|
+
|
16
|
+
class << self
|
17
|
+
|
18
|
+
# Returns an unbound method object that represents the method that
|
19
|
+
# should be called to dispose of +rsrc+.
|
20
|
+
def disposal_method(rsrc)
|
21
|
+
disp_mthd = resource_types[rsrc.class] || resource_types.find{ |rt, _| rt === rsrc }.last rescue nil
|
22
|
+
disp_mthd or raise ArgumentError, "No registered resource class matches '#{rsrc.class}'"
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
# Register a resource type for automated resource management.
|
27
|
+
#
|
28
|
+
# @param rsrc_cls [Class] The class of resource to be managed. This
|
29
|
+
# must be the type of the object that will be returned when an
|
30
|
+
# instance of this resource is acquired.
|
31
|
+
# @param helper_mthd [Symbol] The name of the resource acquisition
|
32
|
+
# helper method that should be added to the ResourceHelper module.
|
33
|
+
# @param options [Hash] An options class.
|
34
|
+
# @option options [Symbol] :acquisition_method For cases where an
|
35
|
+
# existing method can be called directly on the +rsrc_cls+ to
|
36
|
+
# obtain a resource (rather than passing in a block containing
|
37
|
+
# resource acquisition steps), the name of that method. Defaults
|
38
|
+
# to :open.
|
39
|
+
# @option options [Symbol] :disposal_method The name of the method
|
40
|
+
# to be called on the resource to dispose of it. Defaults to
|
41
|
+
# :close.
|
42
|
+
def register(rsrc_cls, helper_mthd, options = {}, &body)
|
43
|
+
if ResourceHelper.method_defined?(helper_mthd)
|
44
|
+
raise ArgumentError, "Resource acquisition method #{helper_mthd} is already registered"
|
45
|
+
end
|
46
|
+
unless body
|
47
|
+
open_mthd = options.fetch(:acquisition_method, :open)
|
48
|
+
body = lambda{ |*args| rsrc_cls.send(open_mthd, *args) }
|
49
|
+
end
|
50
|
+
disp_mthd = options.fetch(:disposal_method, :close)
|
51
|
+
|
52
|
+
if rsrc_cls.method_defined?(disp_mthd)
|
53
|
+
if (m = resource_types[rsrc_cls]) && m.name != disp_mthd
|
54
|
+
raise ArgumentError, "Resource class #{rsrc_cls} has already been registered" +
|
55
|
+
" with a different disposal method (##{m.name})"
|
56
|
+
else
|
57
|
+
resource_types[rsrc_cls] = rsrc_cls.instance_method(disp_mthd)
|
58
|
+
end
|
59
|
+
else
|
60
|
+
raise ArgumentError, "No method named '#{disp_mthd}' is defined on #{rsrc_cls}"
|
61
|
+
end
|
62
|
+
|
63
|
+
# Define the helper method on the ResourceHelper module. This is
|
64
|
+
# necessary (as opposed to just calling the block from the
|
65
|
+
# acquisition methd) in order to ensure that self etc are set
|
66
|
+
# correctly
|
67
|
+
ResourceHelper.class_eval{ define_method(helper_mthd, &body) }
|
68
|
+
|
69
|
+
# Now wrap an aspect around the method to handle the tracking of
|
70
|
+
# resources acquired, and event notifications
|
71
|
+
add_aspect(rsrc_cls, helper_mthd, disp_mthd)
|
72
|
+
Events.publish(self, 'resource.registered', rsrc_cls, helper_mthd)
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
|
79
|
+
def resource_types
|
80
|
+
@resource_types ||= {}
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
# Define the helper method to acquire a resource, publish events about
|
85
|
+
# the resource lifecycle, and track the usage of the resource to
|
86
|
+
# ensure we know about unreleased resources and can clean then up at
|
87
|
+
# the appropriate time when the owning object is done with them.
|
88
|
+
def add_aspect(rsrc_cls, helper_mthd, disp_mthd)
|
89
|
+
mthd = ResourceHelper.instance_method(helper_mthd)
|
90
|
+
ResourceHelper.class_eval do
|
91
|
+
define_method helper_mthd do |*args|
|
92
|
+
if Events.publish(rsrc_cls, 'resource.pre_acquire', *args)
|
93
|
+
result = nil
|
94
|
+
begin
|
95
|
+
result = mthd.bind(self).call(*args)
|
96
|
+
unless rsrc_cls === result
|
97
|
+
raise ArgumentError, "Returned resource is of type #{
|
98
|
+
result.class.name}, not #{rsrc_cls}"
|
99
|
+
end
|
100
|
+
# Override disposal method on this acquired instance
|
101
|
+
# to call #dispose_resource instead
|
102
|
+
defn = self
|
103
|
+
result.define_singleton_method(disp_mthd) do
|
104
|
+
defn.dispose_resource(self)
|
105
|
+
end
|
106
|
+
add_resource(result)
|
107
|
+
Events.publish(rsrc_cls, 'resource.acquired', result)
|
108
|
+
result
|
109
|
+
rescue Exception => ex
|
110
|
+
Events.publish(rsrc_cls, 'resource.acquisition_failed', ex)
|
111
|
+
raise
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
|
124
|
+
# A module that can be included in a class to provide resource acquisition
|
125
|
+
# with automated resource cleanup.
|
126
|
+
#
|
127
|
+
# Resources acquired via this module are tracked, and can be disposed of
|
128
|
+
# when no longer needed via a call to the #cleanup_resources method.
|
129
|
+
#
|
130
|
+
# The benefits of including and using ResourceHelper module:
|
131
|
+
# - Resource acquisition can be setup to use a common configuration process,
|
132
|
+
# such as obtaining connection details from a shared configuration file.
|
133
|
+
# - All resources obtained by an object can be freed when the object is
|
134
|
+
# done with them by calling the #cleanup_resources.
|
135
|
+
module ResourceHelper
|
136
|
+
|
137
|
+
# Register a resource for later clean-up
|
138
|
+
def add_resource(rsrc)
|
139
|
+
# Ensure we know how to dispose of this resource
|
140
|
+
ResourceManager.disposal_method(rsrc)
|
141
|
+
(@__resources__ ||= Set.new) << rsrc
|
142
|
+
end
|
143
|
+
|
144
|
+
|
145
|
+
# Dispose of a resource.
|
146
|
+
#
|
147
|
+
# This method will be called automatically whenever a resource is closed
|
148
|
+
# manually (via a call to the resources normal disposal method, e.g.
|
149
|
+
# #close), or when #cleanup_resources is used to tidy-up all managed
|
150
|
+
# resources.
|
151
|
+
def dispose_resource(rsrc)
|
152
|
+
disp_mthd = ResourceManager.disposal_method(rsrc)
|
153
|
+
@__resources__.delete(rsrc)
|
154
|
+
if Events.publish(rsrc, 'resource.pre-disposal')
|
155
|
+
begin
|
156
|
+
disp_mthd.bind(rsrc).call
|
157
|
+
Events.publish(rsrc, 'resource.disposed')
|
158
|
+
rescue Exception => ex
|
159
|
+
Events.publish(rsrc, 'resource.disposal-failed', ex)
|
160
|
+
raise
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
|
166
|
+
# Dispose of all resources managed by this object.
|
167
|
+
def cleanup_resources
|
168
|
+
if @__resources__
|
169
|
+
@__resources__.clone.reverse_each do |rsrc|
|
170
|
+
dispose_resource(rsrc)
|
171
|
+
end
|
172
|
+
@__resources__ = nil
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
|
177
|
+
# Add automatic disposal of resources on completion of job if included
|
178
|
+
# into a job.
|
179
|
+
def self.included(cls)
|
180
|
+
if (defined?(BatchKit::Job) && BatchKit::Job == cls) ||
|
181
|
+
(defined?(BatchKit::ActsAsJob) && cls.include?(BatchKit::ActsAsJob))
|
182
|
+
Events.subscribe(BatchKit::Job::Run, 'post-execute') do |run, job_obj, ok|
|
183
|
+
job_obj.cleanup_resources
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
end
|
189
|
+
|
190
|
+
end
|
191
|
+
|