batch-kit 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +22 -0
  3. data/README.md +165 -0
  4. data/lib/batch-kit.rb +9 -0
  5. data/lib/batch-kit/arguments.rb +57 -0
  6. data/lib/batch-kit/config.rb +517 -0
  7. data/lib/batch-kit/configurable.rb +68 -0
  8. data/lib/batch-kit/core_ext/enumerable.rb +97 -0
  9. data/lib/batch-kit/core_ext/file.rb +69 -0
  10. data/lib/batch-kit/core_ext/file_utils.rb +103 -0
  11. data/lib/batch-kit/core_ext/hash.rb +17 -0
  12. data/lib/batch-kit/core_ext/numeric.rb +17 -0
  13. data/lib/batch-kit/core_ext/string.rb +88 -0
  14. data/lib/batch-kit/database.rb +133 -0
  15. data/lib/batch-kit/database/java_util_log_handler.rb +65 -0
  16. data/lib/batch-kit/database/log4r_outputter.rb +57 -0
  17. data/lib/batch-kit/database/models.rb +548 -0
  18. data/lib/batch-kit/database/schema.rb +229 -0
  19. data/lib/batch-kit/encryption.rb +7 -0
  20. data/lib/batch-kit/encryption/java_encryption.rb +178 -0
  21. data/lib/batch-kit/encryption/ruby_encryption.rb +175 -0
  22. data/lib/batch-kit/events.rb +157 -0
  23. data/lib/batch-kit/framework/acts_as_job.rb +197 -0
  24. data/lib/batch-kit/framework/acts_as_sequence.rb +123 -0
  25. data/lib/batch-kit/framework/definable.rb +169 -0
  26. data/lib/batch-kit/framework/job.rb +121 -0
  27. data/lib/batch-kit/framework/job_definition.rb +105 -0
  28. data/lib/batch-kit/framework/job_run.rb +145 -0
  29. data/lib/batch-kit/framework/runnable.rb +235 -0
  30. data/lib/batch-kit/framework/sequence.rb +87 -0
  31. data/lib/batch-kit/framework/sequence_definition.rb +38 -0
  32. data/lib/batch-kit/framework/sequence_run.rb +48 -0
  33. data/lib/batch-kit/framework/task_definition.rb +89 -0
  34. data/lib/batch-kit/framework/task_run.rb +53 -0
  35. data/lib/batch-kit/helpers/date_time.rb +54 -0
  36. data/lib/batch-kit/helpers/email.rb +198 -0
  37. data/lib/batch-kit/helpers/html.rb +175 -0
  38. data/lib/batch-kit/helpers/process.rb +101 -0
  39. data/lib/batch-kit/helpers/zip.rb +30 -0
  40. data/lib/batch-kit/job.rb +11 -0
  41. data/lib/batch-kit/lockable.rb +138 -0
  42. data/lib/batch-kit/loggable.rb +78 -0
  43. data/lib/batch-kit/logging.rb +169 -0
  44. data/lib/batch-kit/logging/java_util_logger.rb +87 -0
  45. data/lib/batch-kit/logging/log4r_logger.rb +71 -0
  46. data/lib/batch-kit/logging/null_logger.rb +35 -0
  47. data/lib/batch-kit/logging/stdout_logger.rb +96 -0
  48. data/lib/batch-kit/resources.rb +191 -0
  49. data/lib/batch-kit/sequence.rb +7 -0
  50. metadata +122 -0
@@ -0,0 +1,87 @@
1
+ require 'java'
2
+
3
+
4
+ class BatchKit
5
+
6
+ module Logging
7
+
8
+ class JavaLogFacade
9
+
10
+ LEVEL_MAP = {
11
+ :error => Java::JavaUtilLogging::Level::SEVERE,
12
+ :warning => Java::JavaUtilLogging::Level::WARNING,
13
+ :info => Java::JavaUtilLogging::Level::INFO,
14
+ :config => Java::JavaUtilLogging::Level::CONFIG,
15
+ :detail => Java::JavaUtilLogging::Level::FINE,
16
+ :trace => Java::JavaUtilLogging::Level::FINER,
17
+ :debug => Java::JavaUtilLogging::Level::FINEST
18
+ }
19
+
20
+ # @return The path to any log file used with this logger
21
+ attr_reader :log_file
22
+
23
+
24
+ def initialize(logger)
25
+ @java_logger = logger
26
+ end
27
+
28
+
29
+ def level
30
+ LEVEL_MAP.invert[@java_logger.getLevel()]
31
+ end
32
+
33
+
34
+ def level=(level)
35
+ @java_logger.setLevel(LEVEL_MAP[level])
36
+ end
37
+
38
+
39
+ # Adds a FileHandler to capture output from this logger to a log file.
40
+ def log_file=(log_path)
41
+ @java_logger.getHandlers().each do |h|
42
+ if h.is_a?(Java::JavaUtilLogging::FileHandler)
43
+ @java_logger.removeHandler(h)
44
+ h.close()
45
+ end
46
+ end
47
+ @log_file = log_path
48
+ if log_path
49
+ # Java logger does not follow changes in working directory via Dir.chdir
50
+ log_path = File.absolute_path(log_path)
51
+ FileUtils.mkdir_p(File.dirname(log_path))
52
+ fh = Java::JavaUtilLogging::FileHandler.new(log_path, true)
53
+ if defined?(Console::JavaUtilLogger)
54
+ fmt = Console::JavaUtilLogger::RubyFormatter.new('[%1$tF %1$tT] %4$-6s %5$s', -1)
55
+ fmt.level_labels[Java::JavaUtilLogging::Level::FINE] = 'DETAIL'
56
+ fmt.level_labels[Java::JavaUtilLogging::Level::FINER] = 'TRACE'
57
+ else
58
+ fmt = Java::JavaUtilLogging::SimpleFormatter.new
59
+ end
60
+ fh.setFormatter(fmt)
61
+ self.addHandler(fh)
62
+ end
63
+ end
64
+
65
+
66
+ BatchKit::Logging::LEVELS.each do |lvl|
67
+ java_mthd = LEVEL_MAP[lvl].getName().downcase.intern
68
+ class_eval <<-EOD
69
+ def #{lvl}(msg)
70
+ @java_logger.#{java_mthd}(msg.to_s)
71
+ end
72
+ EOD
73
+ end
74
+
75
+ alias_method :warn, :warning
76
+
77
+
78
+ def method_missing(mthd, *args)
79
+ @java_logger.send(mthd, *args)
80
+ end
81
+
82
+ end
83
+
84
+ end
85
+
86
+ end
87
+
@@ -0,0 +1,71 @@
1
+ require 'log4r'
2
+ require 'log4r/configurator'
3
+
4
+ Log4r::Configurator.custom_levels(*BatchKit::Logging::LEVELS.reverse.map{ |l| l.to_s.upcase })
5
+
6
+
7
+
8
+ class BatchKit
9
+
10
+ module Logging
11
+
12
+ class Log4rFacade
13
+
14
+ def initialize(logger)
15
+ @log4r_logger = logger
16
+ end
17
+
18
+
19
+ def level
20
+ Log4r::LNAMES[@log4r_logger.level].downcase.intern
21
+ end
22
+
23
+
24
+ def level=(lvl)
25
+ @log4r_logger.level = Log4r::LNAMES.index(lvl.to_s.upcase)
26
+ end
27
+
28
+
29
+ def log_file
30
+ out_name = "#{self.name}_file"
31
+ fo = @log4r_logger.outputters.find{ |o| o.name == out_name }
32
+ fo && fo.filename
33
+ end
34
+
35
+
36
+ def log_file=(log_path)
37
+ out_name = "#{self.name}_file"
38
+ if outputter = Log4r::Outputter[out_name]
39
+ outputter.close
40
+ @log4r_logger.remove out_name
41
+ end
42
+ if log_path
43
+ FileUtils.mkdir_p(File.dirname(log_path))
44
+ formatter = Log4r::PatternFormatter.new(pattern: '[%d] %-6l %x %M\r')
45
+ outputter = Log4r::FileOutputter.new(out_name, filename: log_path,
46
+ trunc: false, formatter: formatter)
47
+ @log4r_logger.add out_name
48
+ end
49
+ end
50
+
51
+
52
+ BatchKit::Logging::LEVELS.each do |lvl|
53
+ class_eval <<-EOD
54
+ def #{lvl}(msg)
55
+ @log4r_logger.#{lvl}(msg)
56
+ end
57
+ EOD
58
+ end
59
+
60
+ alias_method :warn, :warning
61
+
62
+
63
+ def method_missing(mthd, *args)
64
+ @log4r_logger.send(mthd, *args)
65
+ end
66
+
67
+ end
68
+
69
+ end
70
+
71
+ end
@@ -0,0 +1,35 @@
1
+ class BatchKit
2
+
3
+ module Logging
4
+
5
+ # Implements a NULL logger, i.e. a logger that throws away log messages.
6
+ # This logger should be used when no logging is desired.
7
+ class NullLogger
8
+
9
+
10
+ def self.instance
11
+ @instance ||= self.new
12
+ end
13
+
14
+
15
+ def log_level=(level)
16
+ end
17
+
18
+
19
+ LEVELS.each do |level|
20
+ define_method level do |*args|
21
+ log_msg(level, *args)
22
+ end
23
+ end
24
+
25
+ alias_method :warn, :warning
26
+
27
+
28
+ def log_msg(level, msg)
29
+ end
30
+
31
+ end
32
+
33
+ end
34
+
35
+ end
@@ -0,0 +1,96 @@
1
+ class BatchKit
2
+
3
+ module Logging
4
+
5
+ class StdOutLogger
6
+
7
+ def self.logger(name)
8
+ @loggers ||= {}
9
+ @loggers[name] ||= self.new(name)
10
+ end
11
+
12
+
13
+ # @return [String] The name of this logger
14
+ attr_reader :name
15
+ # @return [Symbol] The current level at which logging is set
16
+ attr_accessor :level
17
+ # @return [String] The log file path, if any
18
+ attr_reader :log_file
19
+
20
+ # Width at which to split lines
21
+ attr_accessor :width
22
+ # Amount by which to indent lines
23
+ attr_accessor :indent
24
+
25
+
26
+ def initialize(name, level = :detail)
27
+ @name = name
28
+ @level = level
29
+ @indent = 8
30
+ @width = Console.width if use_console?
31
+ end
32
+
33
+
34
+ LEVELS.each do |level|
35
+ define_method level do |*args|
36
+ log_msg(level, *args)
37
+ end
38
+ end
39
+
40
+ alias_method :warn, :warning
41
+
42
+
43
+ def log_file=(log_path, options = {})
44
+ @log_file.close if @log_file
45
+ if log_path
46
+ append = options.fetch(:append, true)
47
+ @log_file = File.new(log_path, append ? 'a' : 'w')
48
+ end
49
+ end
50
+
51
+
52
+ def log_msg(level, *args)
53
+ return if LEVELS.index(level) > LEVELS.index(@level)
54
+ lvl = level.to_s.upcase
55
+ msg = args.join(' ')
56
+ spacer = LEVELS.index(level) >= LEVELS.index(:config) ? ' ' : ''
57
+ fmt_msg = "%-6s %s%s" % [lvl, spacer, msg]
58
+ if use_console?
59
+ color = case level
60
+ when :error then :red
61
+ when :warning then :yellow
62
+ when :info then :white
63
+ when :config then :cyan
64
+ when :detail then :light_gray
65
+ else :dark_gray
66
+ end
67
+
68
+ indent = @indent || 0
69
+ indent += 2 if indent > 0 && [:config, :detail, :trace, :debug].include?(level)
70
+
71
+ msg = @width ? Console.wrap_text(msg, @width - indent) : [msg]
72
+ msg = msg.each_with_index.map do |line, i|
73
+ "%-6s %s%s" % [[lvl][i], spacer, line]
74
+ end.join("\n")
75
+ Console.puts msg, color
76
+ else
77
+ STDOUT.puts fmt_msg
78
+ end
79
+ if @log_file
80
+ @log_file.puts Time.now.strftime('[%F %T] ') + fmt_msg
81
+ end
82
+ end
83
+
84
+
85
+ def use_console?
86
+ unless @use_console
87
+ @use_console = defined?(::Console)
88
+ end
89
+ @use_console
90
+ end
91
+
92
+ end
93
+
94
+ end
95
+
96
+ end
@@ -0,0 +1,191 @@
1
+ require 'set'
2
+ require_relative 'events'
3
+
4
+
5
+ class BatchKit
6
+
7
+ # Defines a manager for resource types, such as database connections etc.
8
+ # Resource types are registered with this class, which then adds acquisition
9
+ # methods to the ResourceHelper module. These acquisition methods add the
10
+ # acquired objects to a collection managed by the objects of the class that
11
+ # includes the ResourceHelper, and modify the returned resource objects so
12
+ # that they automatically de-register themselves if they are disposed of
13
+ # explicitly.
14
+ class ResourceManager
15
+
16
+ class << self
17
+
18
+ # Returns an unbound method object that represents the method that
19
+ # should be called to dispose of +rsrc+.
20
+ def disposal_method(rsrc)
21
+ disp_mthd = resource_types[rsrc.class] || resource_types.find{ |rt, _| rt === rsrc }.last rescue nil
22
+ disp_mthd or raise ArgumentError, "No registered resource class matches '#{rsrc.class}'"
23
+ end
24
+
25
+
26
+ # Register a resource type for automated resource management.
27
+ #
28
+ # @param rsrc_cls [Class] The class of resource to be managed. This
29
+ # must be the type of the object that will be returned when an
30
+ # instance of this resource is acquired.
31
+ # @param helper_mthd [Symbol] The name of the resource acquisition
32
+ # helper method that should be added to the ResourceHelper module.
33
+ # @param options [Hash] An options class.
34
+ # @option options [Symbol] :acquisition_method For cases where an
35
+ # existing method can be called directly on the +rsrc_cls+ to
36
+ # obtain a resource (rather than passing in a block containing
37
+ # resource acquisition steps), the name of that method. Defaults
38
+ # to :open.
39
+ # @option options [Symbol] :disposal_method The name of the method
40
+ # to be called on the resource to dispose of it. Defaults to
41
+ # :close.
42
+ def register(rsrc_cls, helper_mthd, options = {}, &body)
43
+ if ResourceHelper.method_defined?(helper_mthd)
44
+ raise ArgumentError, "Resource acquisition method #{helper_mthd} is already registered"
45
+ end
46
+ unless body
47
+ open_mthd = options.fetch(:acquisition_method, :open)
48
+ body = lambda{ |*args| rsrc_cls.send(open_mthd, *args) }
49
+ end
50
+ disp_mthd = options.fetch(:disposal_method, :close)
51
+
52
+ if rsrc_cls.method_defined?(disp_mthd)
53
+ if (m = resource_types[rsrc_cls]) && m.name != disp_mthd
54
+ raise ArgumentError, "Resource class #{rsrc_cls} has already been registered" +
55
+ " with a different disposal method (##{m.name})"
56
+ else
57
+ resource_types[rsrc_cls] = rsrc_cls.instance_method(disp_mthd)
58
+ end
59
+ else
60
+ raise ArgumentError, "No method named '#{disp_mthd}' is defined on #{rsrc_cls}"
61
+ end
62
+
63
+ # Define the helper method on the ResourceHelper module. This is
64
+ # necessary (as opposed to just calling the block from the
65
+ # acquisition methd) in order to ensure that self etc are set
66
+ # correctly
67
+ ResourceHelper.class_eval{ define_method(helper_mthd, &body) }
68
+
69
+ # Now wrap an aspect around the method to handle the tracking of
70
+ # resources acquired, and event notifications
71
+ add_aspect(rsrc_cls, helper_mthd, disp_mthd)
72
+ Events.publish(self, 'resource.registered', rsrc_cls, helper_mthd)
73
+ end
74
+
75
+
76
+ private
77
+
78
+
79
+ def resource_types
80
+ @resource_types ||= {}
81
+ end
82
+
83
+
84
+ # Define the helper method to acquire a resource, publish events about
85
+ # the resource lifecycle, and track the usage of the resource to
86
+ # ensure we know about unreleased resources and can clean then up at
87
+ # the appropriate time when the owning object is done with them.
88
+ def add_aspect(rsrc_cls, helper_mthd, disp_mthd)
89
+ mthd = ResourceHelper.instance_method(helper_mthd)
90
+ ResourceHelper.class_eval do
91
+ define_method helper_mthd do |*args|
92
+ if Events.publish(rsrc_cls, 'resource.pre_acquire', *args)
93
+ result = nil
94
+ begin
95
+ result = mthd.bind(self).call(*args)
96
+ unless rsrc_cls === result
97
+ raise ArgumentError, "Returned resource is of type #{
98
+ result.class.name}, not #{rsrc_cls}"
99
+ end
100
+ # Override disposal method on this acquired instance
101
+ # to call #dispose_resource instead
102
+ defn = self
103
+ result.define_singleton_method(disp_mthd) do
104
+ defn.dispose_resource(self)
105
+ end
106
+ add_resource(result)
107
+ Events.publish(rsrc_cls, 'resource.acquired', result)
108
+ result
109
+ rescue Exception => ex
110
+ Events.publish(rsrc_cls, 'resource.acquisition_failed', ex)
111
+ raise
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
117
+
118
+ end
119
+
120
+ end
121
+
122
+
123
+
124
+ # A module that can be included in a class to provide resource acquisition
125
+ # with automated resource cleanup.
126
+ #
127
+ # Resources acquired via this module are tracked, and can be disposed of
128
+ # when no longer needed via a call to the #cleanup_resources method.
129
+ #
130
+ # The benefits of including and using ResourceHelper module:
131
+ # - Resource acquisition can be setup to use a common configuration process,
132
+ # such as obtaining connection details from a shared configuration file.
133
+ # - All resources obtained by an object can be freed when the object is
134
+ # done with them by calling the #cleanup_resources.
135
+ module ResourceHelper
136
+
137
+ # Register a resource for later clean-up
138
+ def add_resource(rsrc)
139
+ # Ensure we know how to dispose of this resource
140
+ ResourceManager.disposal_method(rsrc)
141
+ (@__resources__ ||= Set.new) << rsrc
142
+ end
143
+
144
+
145
+ # Dispose of a resource.
146
+ #
147
+ # This method will be called automatically whenever a resource is closed
148
+ # manually (via a call to the resources normal disposal method, e.g.
149
+ # #close), or when #cleanup_resources is used to tidy-up all managed
150
+ # resources.
151
+ def dispose_resource(rsrc)
152
+ disp_mthd = ResourceManager.disposal_method(rsrc)
153
+ @__resources__.delete(rsrc)
154
+ if Events.publish(rsrc, 'resource.pre-disposal')
155
+ begin
156
+ disp_mthd.bind(rsrc).call
157
+ Events.publish(rsrc, 'resource.disposed')
158
+ rescue Exception => ex
159
+ Events.publish(rsrc, 'resource.disposal-failed', ex)
160
+ raise
161
+ end
162
+ end
163
+ end
164
+
165
+
166
+ # Dispose of all resources managed by this object.
167
+ def cleanup_resources
168
+ if @__resources__
169
+ @__resources__.clone.reverse_each do |rsrc|
170
+ dispose_resource(rsrc)
171
+ end
172
+ @__resources__ = nil
173
+ end
174
+ end
175
+
176
+
177
+ # Add automatic disposal of resources on completion of job if included
178
+ # into a job.
179
+ def self.included(cls)
180
+ if (defined?(BatchKit::Job) && BatchKit::Job == cls) ||
181
+ (defined?(BatchKit::ActsAsJob) && cls.include?(BatchKit::ActsAsJob))
182
+ Events.subscribe(BatchKit::Job::Run, 'post-execute') do |run, job_obj, ok|
183
+ job_obj.cleanup_resources
184
+ end
185
+ end
186
+ end
187
+
188
+ end
189
+
190
+ end
191
+