tap 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Basic Overview +151 -0
- data/Command Reference +99 -0
- data/History +24 -0
- data/MIT-LICENSE +1 -1
- data/README +29 -57
- data/Rakefile +30 -37
- data/Tutorial +243 -191
- data/bin/tap +66 -35
- data/lib/tap.rb +47 -29
- data/lib/tap/app.rb +700 -342
- data/lib/tap/{script → cmd}/console.rb +0 -0
- data/lib/tap/{script → cmd}/destroy.rb +0 -0
- data/lib/tap/{script → cmd}/generate.rb +0 -0
- data/lib/tap/cmd/run.rb +156 -0
- data/lib/tap/constants.rb +4 -0
- data/lib/tap/dump.rb +57 -0
- data/lib/tap/env.rb +316 -0
- data/lib/tap/file_task.rb +106 -109
- data/lib/tap/generator.rb +4 -1
- data/lib/tap/generator/generators/command/USAGE +6 -0
- data/lib/tap/generator/generators/command/command_generator.rb +17 -0
- data/lib/tap/generator/generators/{script/templates/script.erb → command/templates/command.erb} +10 -10
- data/lib/tap/generator/generators/config/USAGE +21 -0
- data/lib/tap/generator/generators/config/config_generator.rb +17 -7
- data/lib/tap/generator/generators/file_task/USAGE +3 -0
- data/lib/tap/generator/generators/file_task/file_task_generator.rb +16 -0
- data/lib/tap/generator/generators/file_task/templates/file.txt +2 -0
- data/lib/tap/generator/generators/file_task/templates/file.yml +3 -0
- data/lib/tap/generator/generators/file_task/templates/task.erb +26 -20
- data/lib/tap/generator/generators/file_task/templates/test.erb +20 -10
- data/lib/tap/generator/generators/generator/generator_generator.rb +1 -1
- data/lib/tap/generator/generators/generator/templates/generator.erb +21 -12
- data/lib/tap/generator/generators/root/templates/Rakefile +33 -24
- data/lib/tap/generator/generators/root/templates/tap.yml +28 -31
- data/lib/tap/generator/generators/root/templates/test/tap_test_helper.rb +1 -0
- data/lib/tap/generator/generators/task/USAGE +3 -0
- data/lib/tap/generator/generators/task/task_generator.rb +18 -5
- data/lib/tap/generator/generators/task/templates/task.erb +7 -12
- data/lib/tap/generator/generators/task/templates/test.erb +10 -11
- data/lib/tap/generator/generators/workflow/templates/task.erb +1 -1
- data/lib/tap/generator/generators/workflow/templates/test.erb +1 -1
- data/lib/tap/patches/rake/rake_test_loader.rb +8 -0
- data/lib/tap/patches/rake/testtask.rb +55 -0
- data/lib/tap/patches/ruby19/backtrace_filter.rb +51 -0
- data/lib/tap/patches/ruby19/parsedate.rb +16 -0
- data/lib/tap/root.rb +172 -67
- data/lib/tap/script.rb +70 -336
- data/lib/tap/support/aggregator.rb +55 -0
- data/lib/tap/support/audit.rb +281 -280
- data/lib/tap/support/batchable.rb +59 -0
- data/lib/tap/support/class_configuration.rb +279 -0
- data/lib/tap/support/configurable.rb +92 -0
- data/lib/tap/support/configurable_methods.rb +296 -0
- data/lib/tap/support/executable.rb +98 -0
- data/lib/tap/support/executable_queue.rb +82 -0
- data/lib/tap/support/logger.rb +9 -15
- data/lib/tap/support/rake.rb +43 -54
- data/lib/tap/support/run_error.rb +32 -13
- data/lib/tap/support/shell_utils.rb +47 -0
- data/lib/tap/support/tdoc.rb +9 -8
- data/lib/tap/support/tdoc/config_attr.rb +40 -16
- data/lib/tap/support/validation.rb +77 -0
- data/lib/tap/support/versions.rb +36 -36
- data/lib/tap/task.rb +276 -482
- data/lib/tap/test.rb +20 -261
- data/lib/tap/test/env_vars.rb +7 -5
- data/lib/tap/test/file_methods.rb +126 -121
- data/lib/tap/test/subset_methods.rb +86 -45
- data/lib/tap/test/tap_methods.rb +271 -0
- data/lib/tap/workflow.rb +174 -46
- data/test/app/config/another/task.yml +1 -0
- data/test/app/config/erb.yml +2 -1
- data/test/app/config/some/task.yml +1 -0
- data/test/app/config/template.yml +2 -6
- data/test/app_test.rb +1241 -1008
- data/test/env/test_configure/recurse_a.yml +2 -0
- data/test/env/test_configure/recurse_b.yml +2 -0
- data/test/env/test_configure/tap.yml +23 -0
- data/test/env/test_load_env_config/dir/tap.yml +3 -0
- data/test/env/test_load_env_config/recurse_a.yml +2 -0
- data/test/env/test_load_env_config/recurse_b.yml +2 -0
- data/test/env/test_load_env_config/tap.yml +3 -0
- data/test/env_test.rb +198 -0
- data/test/file_task_test.rb +70 -53
- data/{lib/tap/generator/generators/package/USAGE → test/root/file.txt} +0 -0
- data/test/root_test.rb +621 -454
- data/test/script_test.rb +38 -174
- data/test/support/aggregator_test.rb +99 -0
- data/test/support/audit_test.rb +409 -416
- data/test/support/batchable_test.rb +74 -0
- data/test/support/{task_configuration_test.rb → class_configuration_test.rb} +106 -47
- data/test/{task/config/overriding.yml → support/configurable/config/configured.yml} +0 -0
- data/test/support/configurable_test.rb +295 -0
- data/test/support/executable_queue_test.rb +103 -0
- data/test/support/executable_test.rb +38 -0
- data/test/support/logger_test.rb +17 -17
- data/test/support/rake_test.rb +4 -2
- data/test/support/shell_utils_test.rb +24 -0
- data/test/support/tdoc_test.rb +265 -258
- data/test/support/validation_test.rb +54 -0
- data/test/support/versions_test.rb +38 -38
- data/test/tap_test_helper.rb +19 -5
- data/test/tap_test_suite.rb +5 -2
- data/test/task_base_test.rb +13 -104
- data/test/task_syntax_test.rb +300 -0
- data/test/task_test.rb +258 -381
- data/test/test/env_vars_test.rb +40 -40
- data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/one.txt +0 -0
- data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/two.txt +0 -0
- data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/one.txt +0 -0
- data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/two.txt +0 -0
- data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/one.txt +0 -0
- data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/two.txt +0 -0
- data/test/test/file_methods/test_assert_files_fails_for_different_content/expected/one.txt +1 -0
- data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_different_content}/expected/two.txt +0 -0
- data/test/test/file_methods/test_assert_files_fails_for_different_content/input/one.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_different_content/input/two.txt +1 -0
- data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_missing_expected_file}/expected/one.txt +0 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/one.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/two.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/one.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/two.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/one.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/two.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/one.txt +1 -0
- data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/two.txt +1 -0
- data/test/test/file_methods_doc/test_sub/expected/one.txt +1 -0
- data/test/test/file_methods_doc/test_sub/expected/two.txt +1 -0
- data/test/test/file_methods_doc/test_sub/input/one.txt +1 -0
- data/test/test/file_methods_doc/test_sub/input/two.txt +1 -0
- data/test/test/file_methods_doc_test.rb +29 -0
- data/test/test/file_methods_test.rb +214 -143
- data/test/test/subset_methods_test.rb +111 -115
- data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/a.txt +0 -0
- data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/b.txt +0 -0
- data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/a.txt +0 -0
- data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/b.txt +0 -0
- data/test/test/tap_methods_test.rb +399 -0
- data/test/workflow_test.rb +101 -91
- metadata +86 -70
- data/lib/tap/generator/generators/package/package_generator.rb +0 -38
- data/lib/tap/generator/generators/package/templates/package.erb +0 -186
- data/lib/tap/generator/generators/script/USAGE +0 -0
- data/lib/tap/generator/generators/script/script_generator.rb +0 -17
- data/lib/tap/script/run.rb +0 -154
- data/lib/tap/support/batch_queue.rb +0 -162
- data/lib/tap/support/combinator.rb +0 -114
- data/lib/tap/support/task_configuration.rb +0 -169
- data/lib/tap/support/template.rb +0 -81
- data/lib/tap/support/templater.rb +0 -155
- data/lib/tap/version.rb +0 -4
- data/test/app/config/addition_template.yml +0 -6
- data/test/app_class_test.rb +0 -33
- data/test/check/binding_eval.rb +0 -23
- data/test/check/define_method_check.rb +0 -22
- data/test/check/dependencies_check.rb +0 -175
- data/test/check/inheritance_check.rb +0 -22
- data/test/support/batch_queue_test.rb +0 -320
- data/test/support/combinator_test.rb +0 -249
- data/test/support/template_test.rb +0 -122
- data/test/support/templater/erb.txt +0 -2
- data/test/support/templater/erb.yml +0 -2
- data/test/support/templater/somefile.txt +0 -2
- data/test/support/templater_test.rb +0 -192
- data/test/task/config/template.yml +0 -4
- data/test/task_class_test.rb +0 -170
- data/test/task_execute_test.rb +0 -262
- data/test/test/file_methods/test_assert_expected/expected/file.txt +0 -1
- data/test/test/file_methods/test_assert_expected/expected/folder/file.txt +0 -1
- data/test/test/file_methods/test_assert_expected/input/file.txt +0 -1
- data/test/test/file_methods/test_assert_expected/input/folder/file.txt +0 -1
- data/test/test/file_methods/test_assert_files_exist/input/input_1.txt +0 -0
- data/test/test/file_methods/test_assert_files_exist/input/input_2.txt +0 -0
- data/test/test/file_methods/test_file_compare/expected/output_1.txt +0 -3
- data/test/test/file_methods/test_file_compare/expected/output_2.txt +0 -1
- data/test/test/file_methods/test_file_compare/input/input_1.txt +0 -3
- data/test/test/file_methods/test_file_compare/input/input_2.txt +0 -3
- data/test/test/file_methods/test_infer_glob/expected/file.yml +0 -0
- data/test/test/file_methods/test_infer_glob/expected/file_1.txt +0 -0
- data/test/test/file_methods/test_infer_glob/expected/file_2.txt +0 -0
- data/test/test/file_methods/test_yml_compare/expected/output_1.yml +0 -6
- data/test/test/file_methods/test_yml_compare/expected/output_2.yml +0 -6
- data/test/test/file_methods/test_yml_compare/input/input_1.yml +0 -4
- data/test/test/file_methods/test_yml_compare/input/input_2.yml +0 -4
- data/test/test_test.rb +0 -373
data/bin/tap
CHANGED
|
@@ -1,29 +1,60 @@
|
|
|
1
|
+
#!/usr/local/bin/ruby
|
|
1
2
|
# usage: tap <command> {options} [args]
|
|
2
3
|
#
|
|
3
4
|
# examples:
|
|
4
|
-
# tap generate root
|
|
5
|
+
# tap generate root . # generates a root dir
|
|
5
6
|
# tap run taskname --option input # runs the 'taskname' task
|
|
6
7
|
#
|
|
7
8
|
# help:
|
|
8
9
|
# tap help # prints this help
|
|
9
10
|
# tap command --help # prints help for 'command'
|
|
11
|
+
#
|
|
10
12
|
|
|
11
13
|
require File.join( File.dirname(__FILE__), "../lib/tap.rb")
|
|
12
|
-
require "tap/script"
|
|
13
14
|
|
|
15
|
+
# setup environment
|
|
16
|
+
env = Tap::Env.instance
|
|
14
17
|
app = Tap::App.instance
|
|
15
|
-
script = Tap::Script.instance
|
|
16
18
|
|
|
19
|
+
env.logger = app.logger
|
|
20
|
+
if ARGV.delete("-d-")
|
|
21
|
+
env.debug_setup
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
before = nil
|
|
25
|
+
after = nil
|
|
26
|
+
|
|
27
|
+
def handle_error(err)
|
|
28
|
+
case
|
|
29
|
+
when $DEBUG
|
|
30
|
+
puts err.message
|
|
31
|
+
puts
|
|
32
|
+
puts err.backtrace
|
|
33
|
+
when Tap::App.instance.debug? then raise err
|
|
34
|
+
else puts err.message
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# configure the app to tap.yml if it exists
|
|
39
|
+
default_config_file = File.expand_path( Tap::Env::DEFAULT_CONFIG_FILE )
|
|
17
40
|
begin
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
41
|
+
env.load_config(default_config_file, app) do |app, config_file, config|
|
|
42
|
+
unless config_file == default_config_file
|
|
43
|
+
env.log(:warn, "ignoring configs: #{config_file} (#{config.keys.join(',')})", Logger::WARN)
|
|
44
|
+
next
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
before = config.delete('before')
|
|
48
|
+
after = config.delete('after')
|
|
49
|
+
|
|
50
|
+
app.reconfigure(config)
|
|
51
|
+
end
|
|
22
52
|
rescue(Exception)
|
|
23
53
|
# catch errors and exit gracefully
|
|
24
54
|
# (errors usu from gem loading errors)
|
|
25
55
|
puts "Configuration error: #{$!.message}"
|
|
26
|
-
puts
|
|
56
|
+
puts $!.backtrace if $DEBUG
|
|
57
|
+
puts "Check #{default_config_file} configurations"
|
|
27
58
|
exit(1)
|
|
28
59
|
end
|
|
29
60
|
|
|
@@ -36,25 +67,33 @@ end
|
|
|
36
67
|
# run before script
|
|
37
68
|
#
|
|
38
69
|
begin
|
|
39
|
-
eval(
|
|
70
|
+
eval(before.to_s)
|
|
40
71
|
rescue
|
|
41
72
|
puts "Error in before script."
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
else
|
|
45
|
-
puts $!.message
|
|
46
|
-
exit(1)
|
|
47
|
-
end
|
|
73
|
+
handle_error($!)
|
|
74
|
+
exit(1)
|
|
48
75
|
end
|
|
49
76
|
|
|
50
77
|
begin
|
|
51
|
-
available_commands =
|
|
78
|
+
available_commands = env.commands
|
|
52
79
|
command = ARGV.shift
|
|
53
80
|
|
|
54
81
|
case command
|
|
55
82
|
when "--help", "-h", "help", "?", nil
|
|
56
83
|
# give some help
|
|
57
|
-
|
|
84
|
+
File.open(__FILE__) do |file|
|
|
85
|
+
bang_line = true
|
|
86
|
+
file.each_line do |line|
|
|
87
|
+
if bang_line
|
|
88
|
+
bang_line = false
|
|
89
|
+
next
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
break if line !~ /^#\s?(.*)/
|
|
93
|
+
puts $1
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
58
97
|
puts
|
|
59
98
|
puts "available commands:"
|
|
60
99
|
|
|
@@ -64,36 +103,28 @@ begin
|
|
|
64
103
|
print " "
|
|
65
104
|
puts commands.sort.join("\n ")
|
|
66
105
|
puts
|
|
67
|
-
puts "version #{Tap::VERSION} -- #{Tap::
|
|
68
|
-
else
|
|
106
|
+
puts "version #{Tap::VERSION} -- #{Tap::WEBSITE}"
|
|
107
|
+
else
|
|
69
108
|
if available_commands.has_key?(command)
|
|
70
|
-
# run the
|
|
109
|
+
# run the command, if it exists
|
|
71
110
|
load available_commands[command]
|
|
72
111
|
else
|
|
73
|
-
puts "Unknown command: '#{command}'"
|
|
74
|
-
puts "Type 'tap help' for usage information."
|
|
112
|
+
puts "Unknown command: '#{command}'"
|
|
113
|
+
puts "Type 'tap help' for usage information."
|
|
75
114
|
end
|
|
76
115
|
end
|
|
77
116
|
rescue
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
else
|
|
81
|
-
puts $!.message
|
|
82
|
-
puts "Type 'tap #{command} --help' for usage information."
|
|
83
|
-
end
|
|
117
|
+
handle_error($!)
|
|
118
|
+
puts "Type 'tap #{command} --help' for usage information."
|
|
84
119
|
end
|
|
85
120
|
|
|
86
121
|
#
|
|
87
122
|
# run after script
|
|
88
123
|
#
|
|
89
124
|
begin
|
|
90
|
-
eval(
|
|
125
|
+
eval(after.to_s)
|
|
91
126
|
rescue
|
|
92
127
|
puts "Error in after script."
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
else
|
|
96
|
-
puts $!.message
|
|
97
|
-
exit(1)
|
|
98
|
-
end
|
|
128
|
+
handle_error($!)
|
|
129
|
+
exit(1)
|
|
99
130
|
end
|
data/lib/tap.rb
CHANGED
|
@@ -1,44 +1,62 @@
|
|
|
1
|
-
# gem_original_require (set in rubygems) is used here to speed up
|
|
2
|
-
# requires when possible... ok since I'm manually activating the
|
|
3
|
-
# neeeded gems.
|
|
4
1
|
require 'rubygems'
|
|
5
2
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
3
|
+
require 'yaml' # expensive to load
|
|
4
|
+
require 'logger'
|
|
5
|
+
require 'ostruct'
|
|
6
|
+
require 'thread'
|
|
7
|
+
require 'erb'
|
|
8
|
+
|
|
9
|
+
# Apply version-specific patches
|
|
10
|
+
case RUBY_VERSION
|
|
11
|
+
when /^1.9/
|
|
12
|
+
$: << File.dirname(__FILE__) + "/tap/patches/ruby19"
|
|
13
|
+
|
|
14
|
+
# suppresses TDoc warnings
|
|
15
|
+
$DEBUG_RDOC ||= nil
|
|
16
|
+
end
|
|
12
17
|
|
|
13
18
|
# Loading activesupport piecemeal like this cuts the tap load time in half.
|
|
14
19
|
gem 'activesupport'
|
|
15
20
|
|
|
16
|
-
|
|
21
|
+
require 'active_support/core_ext/array/extract_options.rb'
|
|
17
22
|
class Array #:nodoc:
|
|
18
23
|
include ActiveSupport::CoreExtensions::Array::ExtractOptions
|
|
19
24
|
end
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
require 'active_support/core_ext/class.rb'
|
|
26
|
+
require 'active_support/core_ext/module.rb'
|
|
27
|
+
require 'active_support/core_ext/symbol.rb'
|
|
28
|
+
require 'active_support/core_ext/string.rb'
|
|
29
|
+
require 'active_support/core_ext/blank.rb'
|
|
30
|
+
require 'active_support/core_ext/hash/keys.rb'
|
|
31
|
+
require 'active_support/dependencies'
|
|
32
|
+
require 'active_support/clean_logger'
|
|
27
33
|
class Hash #:nodoc:
|
|
28
34
|
include ActiveSupport::CoreExtensions::Hash::Keys
|
|
29
35
|
end
|
|
30
36
|
|
|
31
37
|
$:.unshift File.dirname(__FILE__)
|
|
32
38
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
39
|
+
require 'tap/support/aggregator'
|
|
40
|
+
require 'tap/support/audit'
|
|
41
|
+
require 'tap/support/batchable'
|
|
42
|
+
require 'tap/support/class_configuration'
|
|
43
|
+
require 'tap/support/configurable'
|
|
44
|
+
require 'tap/support/configurable_methods'
|
|
45
|
+
require 'tap/support/executable'
|
|
46
|
+
require 'tap/support/executable_queue'
|
|
47
|
+
require 'tap/support/logger'
|
|
48
|
+
require 'tap/support/run_error'
|
|
49
|
+
require 'tap/support/shell_utils'
|
|
50
|
+
require 'tap/support/validation'
|
|
51
|
+
require 'tap/constants'
|
|
52
|
+
require 'tap/env'
|
|
53
|
+
require 'tap/app'
|
|
54
|
+
require 'tap/task'
|
|
55
|
+
require 'tap/file_task'
|
|
56
|
+
require 'tap/workflow'
|
|
57
|
+
require 'tap/dump'
|
|
58
|
+
|
|
59
|
+
# Apply platform-specific patches
|
|
60
|
+
# case RUBY_PLATFORM
|
|
61
|
+
# when 'java'
|
|
62
|
+
# end
|
data/lib/tap/app.rb
CHANGED
|
@@ -1,63 +1,227 @@
|
|
|
1
1
|
module Tap
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# = Overview
|
|
4
4
|
#
|
|
5
5
|
# App coordinates the setup and running of tasks, and provides an interface
|
|
6
6
|
# to the application directory structure. App is convenient for use within
|
|
7
7
|
# scripts, and provides the basis for the 'tap' command line application.
|
|
8
8
|
#
|
|
9
|
+
# === Task Setup
|
|
10
|
+
#
|
|
9
11
|
# All tasks have an App (by default App.instance) which helps initialize the
|
|
10
|
-
# task by loading configuration templates from the
|
|
11
|
-
#
|
|
12
|
-
#
|
|
12
|
+
# task by loading configuration templates from the config directory. Say
|
|
13
|
+
# we had the following configuration files:
|
|
14
|
+
#
|
|
15
|
+
# [/path/to/app/config/some/task.yml]
|
|
16
|
+
# key: one
|
|
17
|
+
#
|
|
18
|
+
# [/path/to/app/config/another/task.yml]
|
|
19
|
+
# key: two
|
|
20
|
+
#
|
|
21
|
+
# Tasks initialized with the names 'some/task' and 'another/task' will
|
|
22
|
+
# be cofigured by App like this:
|
|
23
|
+
#
|
|
24
|
+
# app = App.instance
|
|
25
|
+
# app.root # => '/path/to/app'
|
|
26
|
+
# app[:config] # => '/path/to/app/config'
|
|
27
|
+
#
|
|
28
|
+
# some_task = Task.new 'some/task'
|
|
29
|
+
# some_task.app # => App.instance
|
|
30
|
+
# some_task.config_file # => '/path/to/app/config/some/task.yml'
|
|
31
|
+
# some_task.config # => {:key => 'one'}
|
|
13
32
|
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
# task.
|
|
33
|
+
# another_task = Task.new 'another/task'
|
|
34
|
+
# another_task.app # => App.instance
|
|
35
|
+
# another_task.config_file # => '/path/to/app/config/another/task.yml'
|
|
36
|
+
# another_task.config # => {:key => 'two'}
|
|
17
37
|
#
|
|
18
|
-
#
|
|
38
|
+
# If app[:config] referenced a different directory then the tasks would be
|
|
39
|
+
# initialized from files relative to that location.
|
|
19
40
|
#
|
|
20
|
-
#
|
|
21
|
-
# audit._current
|
|
22
|
-
# end # => [2,3,4]
|
|
41
|
+
# (see Tap::Root for more details)
|
|
23
42
|
#
|
|
24
43
|
# === Running Tasks
|
|
25
44
|
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
45
|
+
# Task enque commands are passed to app, and tasks access application-wide
|
|
46
|
+
# resources like the logger and options through App.
|
|
47
|
+
#
|
|
48
|
+
# t1 = Task.new {|task, input| input += 1 }
|
|
49
|
+
# t1.enq 0
|
|
50
|
+
# t1.enq 10
|
|
51
|
+
#
|
|
52
|
+
# app.run
|
|
53
|
+
# app.results(t1) # => [1, 11]
|
|
54
|
+
#
|
|
55
|
+
# When a task completes, app collects its results into a data structure that
|
|
56
|
+
# allows access to them as shown above. This behavior can be modified by
|
|
57
|
+
# setting an on_complete block for the task; on_complete blocks can be used
|
|
58
|
+
# to pass results among tasks, allowing the construction of workflows.
|
|
59
|
+
#
|
|
60
|
+
# # clear the previous results
|
|
61
|
+
# app.aggregator.clear
|
|
62
|
+
#
|
|
63
|
+
# t2 = Task.new {|task, input| input += 10 }
|
|
64
|
+
# t1.on_complete {|_result| t2.enq(_result) }
|
|
65
|
+
#
|
|
66
|
+
# t1.enq 0
|
|
67
|
+
# t1.enq 10
|
|
68
|
+
#
|
|
69
|
+
# app.run
|
|
70
|
+
# app.results(t1) # => []
|
|
71
|
+
# app.results(t2) # => [11, 21]
|
|
72
|
+
#
|
|
73
|
+
# Here t1 has no results because the on_complete block passed them to t2 in
|
|
74
|
+
# a simple sequence.
|
|
75
|
+
#
|
|
76
|
+
# === Running Methods
|
|
77
|
+
#
|
|
78
|
+
# Running a task really consists of calling a method. For tasks, the method is
|
|
79
|
+
# basically the block you provide to Task.new, although execution is mediated by
|
|
80
|
+
# Tap::Task#execute and Tap::Task#process so that the block receives the task
|
|
81
|
+
# as a standard input. In subclasses, the method corresponds to the subclass
|
|
82
|
+
# 'process' method.
|
|
83
|
+
#
|
|
84
|
+
# # the block is called to add one to the input
|
|
85
|
+
# Task.new {|task, input| input += 1 }
|
|
86
|
+
#
|
|
87
|
+
# # same thing, but now in a subclass
|
|
88
|
+
# class AddOne < Tap::Task
|
|
89
|
+
# def process(input) input += 1 end
|
|
90
|
+
# end
|
|
91
|
+
#
|
|
92
|
+
# When tasks are enqued, their executable method is pushed onto the queue along
|
|
93
|
+
# with the inputs for the method. Tasks can be batched such that the executable
|
|
94
|
+
# methods of several tasks are enqued at the same time, allowing you to feed the
|
|
95
|
+
# same inputs to multiple methods at once.
|
|
96
|
+
#
|
|
97
|
+
# t1 = Task.new {|task, input| input += 1 }
|
|
98
|
+
# t2 = Task.new {|task, input| input += 10 }
|
|
99
|
+
# Task.batch(t1, t2) # => [t1, t2]
|
|
100
|
+
#
|
|
101
|
+
# t1.enq 0
|
|
102
|
+
# t2.enq 10
|
|
103
|
+
#
|
|
104
|
+
# app.run
|
|
105
|
+
# app.results(t1) # => [1, 11]
|
|
106
|
+
# app.results(t2) # => [10, 20]
|
|
107
|
+
#
|
|
108
|
+
# App also supports multithreading; multithreaded methods execute cosynchronously,
|
|
109
|
+
# each on their own thread (of course, you need to take care to make each method
|
|
110
|
+
# thread safe).
|
|
111
|
+
#
|
|
112
|
+
# lock = Mutex.new
|
|
113
|
+
# array = []
|
|
114
|
+
# t1 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
|
|
115
|
+
# t2 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
|
|
116
|
+
#
|
|
117
|
+
# t1.multithread = true
|
|
118
|
+
# t1.enq
|
|
119
|
+
# t2.multithread = true
|
|
120
|
+
# t2.enq
|
|
121
|
+
#
|
|
122
|
+
# app.run
|
|
123
|
+
# array.length # => 2
|
|
124
|
+
# array[0] == array[1] # => false
|
|
125
|
+
#
|
|
126
|
+
# Since App is geared towards methods, methods from non-task objects can get
|
|
127
|
+
# hooked into a workflow as needed. The preferred way to do so is to make the
|
|
128
|
+
# non-task objects behave like tasks using Task::Base#initialize. The objects
|
|
129
|
+
# can now be enqued, incorporated into workflows, and batched.
|
|
130
|
+
#
|
|
131
|
+
# array = []
|
|
132
|
+
# Task::Base.initialize(array, :push)
|
|
133
|
+
#
|
|
134
|
+
# array.enq(1)
|
|
135
|
+
# array.enq(2)
|
|
29
136
|
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
# thereby allowing results to be dispatched to other tasks.
|
|
137
|
+
# array.empty? # => true
|
|
138
|
+
# app.run
|
|
139
|
+
# array # => [1, 2]
|
|
34
140
|
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
141
|
+
# Lastly, if you can't or don't want to turn your object into a task, Tap defines
|
|
142
|
+
# Object#_method to generate executable objects that can be enqued and
|
|
143
|
+
# incorporated into workflows, although they cannot be batched. The mq
|
|
144
|
+
# (method enq) method generates and enques the method in one step.
|
|
38
145
|
#
|
|
39
|
-
#
|
|
146
|
+
# array = []
|
|
147
|
+
# m = array._method(:push)
|
|
148
|
+
#
|
|
149
|
+
# app.enq(m, 1)
|
|
150
|
+
# app.mq(array, :push, 2)
|
|
40
151
|
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
# code (perhaps performing rollbacks).
|
|
152
|
+
# array.empty? # => true
|
|
153
|
+
# app.run
|
|
154
|
+
# array # => [1, 2]
|
|
45
155
|
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
# and the run exits. If options.debug == true, then the RunError will be
|
|
49
|
-
# raised for further handling.
|
|
156
|
+
# App keeps running as long as it finds methods in the queue, or until it is stopped
|
|
157
|
+
# or terminated.
|
|
50
158
|
#
|
|
51
|
-
#
|
|
52
|
-
# when termination begins and thus will not recieve a TerminationError.
|
|
159
|
+
# (see Tap::Support::Executable, Tap::Task, and Tap::Task::Base for more details)
|
|
53
160
|
#
|
|
54
|
-
#--
|
|
55
161
|
# === Auditing
|
|
56
|
-
|
|
162
|
+
#
|
|
163
|
+
# All results generated by methods are audited to track how a given input
|
|
164
|
+
# evolves during a workflow.
|
|
165
|
+
#
|
|
166
|
+
# To illustrate auditing, consider a workflow that uses the 'add_one' method
|
|
167
|
+
# to add one to an input until the result is 3, then adds five more with the
|
|
168
|
+
# 'add_five' method. The final result should always be 8.
|
|
169
|
+
#
|
|
170
|
+
# t1 = Tap::Task.new('add_one') {|task, input| input += 1 }
|
|
171
|
+
# t2 = Tap::Task.new('add_five') {|task, input| input += 5 }
|
|
172
|
+
#
|
|
173
|
+
# t1.on_complete do |_result|
|
|
174
|
+
# # _result is the audit; use the _current method
|
|
175
|
+
# # to get the current value in the audit trail
|
|
176
|
+
#
|
|
177
|
+
# _result._current < 3 ? t1.enq(_result) : t2.enq(_result)
|
|
178
|
+
# end
|
|
179
|
+
#
|
|
180
|
+
# t1.enq(0)
|
|
181
|
+
# t1.enq(1)
|
|
182
|
+
# t1.enq(2)
|
|
183
|
+
#
|
|
184
|
+
# app.run
|
|
185
|
+
# app.results(t2) # => [8,8,8]
|
|
186
|
+
#
|
|
187
|
+
# Although the results are indistinguishable, each achieved the final value
|
|
188
|
+
# through a different series of tasks. With auditing you can see how each
|
|
189
|
+
# input came to the final value of 8:
|
|
190
|
+
#
|
|
191
|
+
# # app.results returns the actual result values
|
|
192
|
+
# # app._results returns the audits for these values
|
|
193
|
+
# app._results(t2).each do |_result|
|
|
194
|
+
# puts "How #{_result._original} became #{_result._current}:"
|
|
195
|
+
# puts _result._to_s
|
|
196
|
+
# puts
|
|
197
|
+
# end
|
|
198
|
+
#
|
|
199
|
+
# Prints:
|
|
200
|
+
#
|
|
201
|
+
# How 2 became 8:
|
|
202
|
+
# o-[] 2
|
|
203
|
+
# o-[add_one] 3
|
|
204
|
+
# o-[add_five] 8
|
|
205
|
+
#
|
|
206
|
+
# How 1 became 8:
|
|
207
|
+
# o-[] 1
|
|
208
|
+
# o-[add_one] 2
|
|
209
|
+
# o-[add_one] 3
|
|
210
|
+
# o-[add_five] 8
|
|
211
|
+
#
|
|
212
|
+
# How 0 became 8:
|
|
213
|
+
# o-[] 0
|
|
214
|
+
# o-[add_one] 1
|
|
215
|
+
# o-[add_one] 2
|
|
216
|
+
# o-[add_one] 3
|
|
217
|
+
# o-[add_five] 8
|
|
218
|
+
#
|
|
219
|
+
# See Tap::Support::Audit for more details.
|
|
57
220
|
class App < Root
|
|
58
221
|
include MonitorMixin
|
|
59
222
|
|
|
60
223
|
class << self
|
|
224
|
+
# Sets the current app instance
|
|
61
225
|
attr_writer :instance
|
|
62
226
|
|
|
63
227
|
# Returns the current instance of App. If no instance has been set,
|
|
@@ -65,56 +229,62 @@ module Tap
|
|
|
65
229
|
def instance
|
|
66
230
|
@instance ||= App.new
|
|
67
231
|
end
|
|
68
|
-
|
|
69
|
-
# Runs the specified task and inputs with the current instance.
|
|
70
|
-
def run(task, *inputs)
|
|
71
|
-
instance.run(task, *inputs)
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
# Parses the input string as YAML, if the string matches the YAML document
|
|
75
|
-
# specifier (ie it begins with "---\s*\n"). Otherwise returns the string.
|
|
76
|
-
#
|
|
77
|
-
# str = {'key' => 'value'}.to_yaml # => "--- \nkey: value\n"
|
|
78
|
-
# Tap::App.parse_yaml(str) # => {'key' => 'value'}
|
|
79
|
-
# Tap::App.parse_yaml("str") # => "str"
|
|
80
|
-
def parse_yaml(str)
|
|
81
|
-
str =~ /^---\s*\n/ ? YAML.load(str) : str
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
# Read the contents of filepath, templates these using ERB, and loads
|
|
85
|
-
# the result as YAML. Returns nil if filepath does not exist.
|
|
86
|
-
def read_erb_yaml(filepath)
|
|
87
|
-
return nil if !File.exists?(filepath) || File.directory?(filepath)
|
|
88
|
-
|
|
89
|
-
input = File.read(filepath)
|
|
90
|
-
input = ERB.new(input).result
|
|
91
|
-
YAML.load(input)
|
|
92
|
-
end
|
|
93
232
|
end
|
|
94
233
|
|
|
95
|
-
|
|
234
|
+
# An OpenStruct containing the application options.
|
|
235
|
+
attr_reader :options
|
|
236
|
+
|
|
237
|
+
# The shared logger.
|
|
238
|
+
attr_reader :logger
|
|
239
|
+
|
|
240
|
+
# The application queue.
|
|
241
|
+
attr_reader :queue
|
|
242
|
+
|
|
243
|
+
# The state of the application (see App::State).
|
|
244
|
+
attr_reader :state
|
|
245
|
+
|
|
246
|
+
# A hash of (task_name, task_class_name) pairs mapping names to
|
|
247
|
+
# classes for instantiating tasks that have a non-default name.
|
|
248
|
+
# See task_class_name for more details.
|
|
96
249
|
attr_accessor :map
|
|
97
250
|
|
|
98
|
-
#
|
|
251
|
+
# A Tap::Support::Aggregator to collect the results of
|
|
252
|
+
# methods that have no on_complete block.
|
|
253
|
+
attr_reader :aggregator
|
|
254
|
+
|
|
255
|
+
# The constants defining the possible App states.
|
|
99
256
|
module State
|
|
100
257
|
READY = 0
|
|
101
258
|
RUN = 1
|
|
102
259
|
STOP = 2
|
|
103
260
|
TERMINATE = 3
|
|
261
|
+
|
|
262
|
+
module_function
|
|
263
|
+
|
|
264
|
+
# Returns the string corresponding to the input state value.
|
|
265
|
+
# Returns nil for unknown states.
|
|
266
|
+
#
|
|
267
|
+
# State.state_str(0) # => 'READY'
|
|
268
|
+
# State.state_str(12) # => nil
|
|
269
|
+
def state_str(state)
|
|
270
|
+
constants.inject(nil) {|str, s| const_get(s) == state ? s.to_s : str}
|
|
271
|
+
end
|
|
104
272
|
end
|
|
105
273
|
|
|
106
274
|
DEFAULT_MAX_THREADS = 10
|
|
107
|
-
|
|
275
|
+
|
|
108
276
|
# Creates a new App with the given configuration.
|
|
109
277
|
# See reconfigure for configuration options.
|
|
110
278
|
def initialize(config={})
|
|
111
279
|
super()
|
|
112
280
|
|
|
113
|
-
@
|
|
281
|
+
@state = State::READY
|
|
114
282
|
@threads = [].extend(MonitorMixin)
|
|
115
283
|
@thread_queue = nil
|
|
116
|
-
@
|
|
117
|
-
|
|
284
|
+
@run_thread = nil
|
|
285
|
+
|
|
286
|
+
@queue = Support::ExecutableQueue.new
|
|
287
|
+
@aggregator = Support::Aggregator.new
|
|
118
288
|
|
|
119
289
|
# defaults must be provided for options and logging to ensure
|
|
120
290
|
# that they will be initialized by reconfigure
|
|
@@ -122,6 +292,21 @@ module Tap
|
|
|
122
292
|
:options => {}, :logger => {}, :map => {}
|
|
123
293
|
}.merge(config) )
|
|
124
294
|
end
|
|
295
|
+
|
|
296
|
+
# Clears the queue and aggregator.
|
|
297
|
+
#def clear(options={})
|
|
298
|
+
# # syncrhonize?
|
|
299
|
+
# ready
|
|
300
|
+
# raise "cannot clear unless state == READY" unless state == State::READY
|
|
301
|
+
#
|
|
302
|
+
# queue.clear
|
|
303
|
+
# aggregator.clear
|
|
304
|
+
#end
|
|
305
|
+
|
|
306
|
+
# True if options.debug or the global variable $DEBUG is true.
|
|
307
|
+
def debug?
|
|
308
|
+
options.debug || $DEBUG ? true : false
|
|
309
|
+
end
|
|
125
310
|
|
|
126
311
|
# Returns the configuration of self.
|
|
127
312
|
def config
|
|
@@ -149,7 +334,9 @@ module Tap
|
|
|
149
334
|
# Available configurations:
|
|
150
335
|
# root:: resets the root directory of self using root=
|
|
151
336
|
# directories:: resets directory aliases using directories= (note ALL
|
|
152
|
-
# aliases are reset. use app[
|
|
337
|
+
# aliases are reset. use app[dir]= to set a single alias)
|
|
338
|
+
# absolute_paths:: resets absolute path aliases using absolute_paths= (note ALL
|
|
339
|
+
# aliases are reset. use app[dir]= to set a single alias)
|
|
153
340
|
# options:: resets the application options (note ALL options are reset.
|
|
154
341
|
# use app.options.opt= to set a single option)
|
|
155
342
|
# logger:: creates and sets a new logger from the configuration
|
|
@@ -159,9 +346,7 @@ module Tap
|
|
|
159
346
|
# level:: INFO (1)
|
|
160
347
|
# datetime_format:: %H:%M:%S
|
|
161
348
|
#
|
|
162
|
-
#
|
|
163
|
-
# - Unknown configurations raise an error.
|
|
164
|
-
# - Additional configurations may be added in subclasses using handle_configuration
|
|
349
|
+
# Unknown configurations raise an error.
|
|
165
350
|
def reconfigure(config={})
|
|
166
351
|
config = config.symbolize_keys
|
|
167
352
|
|
|
@@ -212,24 +397,89 @@ module Tap
|
|
|
212
397
|
self
|
|
213
398
|
end
|
|
214
399
|
|
|
400
|
+
# Unloads constants loaded by Dependencies, so that they will be reloaded
|
|
401
|
+
# (with any changes made) next time they are called. Returns the unloaded
|
|
402
|
+
# constants.
|
|
403
|
+
def reload
|
|
404
|
+
unloaded = []
|
|
405
|
+
|
|
406
|
+
# echos the behavior of Dependencies.clear,
|
|
407
|
+
# but collects unloaded constants
|
|
408
|
+
Dependencies.loaded.clear
|
|
409
|
+
Dependencies.autoloaded_constants.each do |const|
|
|
410
|
+
Dependencies.remove_constant const
|
|
411
|
+
unloaded << const
|
|
412
|
+
end
|
|
413
|
+
Dependencies.autoloaded_constants.clear
|
|
414
|
+
Dependencies.explicitly_unloadable_constants.each do |const|
|
|
415
|
+
Dependencies.remove_constant const
|
|
416
|
+
unloaded << const
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
unloaded
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
# Looks up the specified constant, dynamically loading via Dependencies
|
|
423
|
+
# if necessary. Returns the const_name if const_name is a Module.
|
|
424
|
+
# Yields to the optional block if the constant cannot be found; otherwise
|
|
425
|
+
# raises a LookupError.
|
|
426
|
+
def lookup_const(const_name)
|
|
427
|
+
return const_name if const_name.kind_of?(Module)
|
|
428
|
+
|
|
429
|
+
begin
|
|
430
|
+
const_name = const_name.camelize
|
|
431
|
+
|
|
432
|
+
case RUBY_VERSION
|
|
433
|
+
when /^1.9/
|
|
434
|
+
|
|
435
|
+
# a check is necessary to maintain the 1.8 behavior
|
|
436
|
+
# of lookup_const in 1.9, where ancestor constants
|
|
437
|
+
# may be returned by a direct evaluation
|
|
438
|
+
const_name.split("::").inject(Object) do |current, const|
|
|
439
|
+
const = const.to_sym
|
|
440
|
+
|
|
441
|
+
current.const_get(const).tap do |c|
|
|
442
|
+
unless current.const_defined?(const, false)
|
|
443
|
+
raise NameError.new("uninitialized constant #{const_name}")
|
|
444
|
+
end
|
|
445
|
+
end
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
else
|
|
449
|
+
Object.module_eval const_name
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
rescue(NameError)
|
|
453
|
+
if block_given?
|
|
454
|
+
yield
|
|
455
|
+
else
|
|
456
|
+
raise LookupError.new("unknown constant: #{const_name}")
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
end
|
|
460
|
+
|
|
215
461
|
#
|
|
216
462
|
# Logging methods
|
|
217
463
|
#
|
|
218
464
|
|
|
219
465
|
# Sets the current logger. The logger is extended with Support::Logger to provide
|
|
220
|
-
# additional logging capabilities.
|
|
466
|
+
# additional logging capabilities. The logger level is set to Logger::DEBUG if
|
|
467
|
+
# the global variable $DEBUG is true.
|
|
221
468
|
def logger=(logger)
|
|
222
469
|
@logger = logger
|
|
223
470
|
@logger.extend Support::Logger unless @logger.nil?
|
|
471
|
+
@logger.level = Logger::DEBUG if $DEBUG
|
|
224
472
|
@logger
|
|
225
473
|
end
|
|
226
474
|
|
|
227
475
|
# Logs the action and message at the input level (default INFO).
|
|
228
|
-
# Logging is suppressed if
|
|
476
|
+
# Logging is suppressed if options.quiet
|
|
229
477
|
def log(action, msg="", level=Logger::INFO)
|
|
230
478
|
logger.add(level, msg, action.to_s) unless options.quiet
|
|
231
479
|
end
|
|
232
480
|
|
|
481
|
+
# EXPERIMENTAL
|
|
482
|
+
#
|
|
233
483
|
# Formatted log. Works like log, but passes the current log format to the
|
|
234
484
|
# block and uses whatever format the block returns. The format recieves
|
|
235
485
|
# the following arguments like so:
|
|
@@ -238,6 +488,8 @@ module Tap
|
|
|
238
488
|
#
|
|
239
489
|
# By default, if you don't specify a block, flog just chomps a newline off
|
|
240
490
|
# the format, so your log will be inline.
|
|
491
|
+
#
|
|
492
|
+
# BUG: Not thread safe at the moment.
|
|
241
493
|
def flog(action="", msg="", level=Logger::INFO) # :yields: format
|
|
242
494
|
unless options.quiet
|
|
243
495
|
logger.format_add(level, msg, action) do |format|
|
|
@@ -245,245 +497,310 @@ module Tap
|
|
|
245
497
|
end
|
|
246
498
|
end
|
|
247
499
|
end
|
|
248
|
-
|
|
249
|
-
# def monitor(action="beginning", msg="", &block)
|
|
250
|
-
# if options.quiet
|
|
251
|
-
# yield
|
|
252
|
-
# nil
|
|
253
|
-
# else
|
|
254
|
-
# @monitoring = true
|
|
255
|
-
# result = logger.monitor(action, msg, &block)
|
|
256
|
-
# @monitoring = false
|
|
257
|
-
# result
|
|
258
|
-
# end
|
|
259
|
-
# end
|
|
260
|
-
#
|
|
261
|
-
# def tick_monitor(action=nil, msg="", level=Logger::INFO)
|
|
262
|
-
# unless options.quiet || !@monitoring
|
|
263
|
-
# action.nil? ?
|
|
264
|
-
# logger.tick :
|
|
265
|
-
# logger.format_add(level, msg, action.to_s) {|format| "\n" + format}
|
|
266
|
-
# end
|
|
267
|
-
# end
|
|
268
|
-
#
|
|
269
|
-
# def monitor_stats(hash)
|
|
270
|
-
# unless options.quiet || !@monitoring
|
|
271
|
-
# logger << "\n"
|
|
272
|
-
# logger << hash.stringify_keys.to_yaml
|
|
273
|
-
# end
|
|
274
|
-
# end
|
|
275
500
|
|
|
276
501
|
#
|
|
277
502
|
# Task methods
|
|
278
503
|
#
|
|
279
504
|
|
|
280
|
-
# Instantiates the specifed task with config (if provided).
|
|
505
|
+
# Instantiates the specifed task with config (if provided). The task
|
|
506
|
+
# class is determined by task_class.
|
|
281
507
|
#
|
|
282
|
-
#
|
|
508
|
+
# t = app.task('tap/file_task')
|
|
509
|
+
# t.class # => Tap::FileTask
|
|
510
|
+
# t.name # => 'tap/file_task'
|
|
283
511
|
#
|
|
512
|
+
# app.map = {"mapped-task" => "Tap::FileTask"}
|
|
284
513
|
# t = app.task('mapped-task-1.0', :key => 'value')
|
|
285
|
-
# t.class # =>
|
|
514
|
+
# t.class # => Tap::FileTask
|
|
286
515
|
# t.name # => "mapped-task-1.0"
|
|
287
516
|
# t.config[:key] # => 'value'
|
|
288
517
|
#
|
|
289
518
|
# A new task is instantiated for each call to task; tasks may share the
|
|
290
|
-
# same name.
|
|
291
|
-
|
|
292
|
-
|
|
519
|
+
# same name.
|
|
520
|
+
def task(task_name, config={}, &block)
|
|
521
|
+
task_class(task_name).new(task_name, config, &block)
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# Looks up the specifed task class. Names are mapped to task classes
|
|
525
|
+
# using task_class_name.
|
|
293
526
|
#
|
|
294
|
-
#
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
527
|
+
# t_class = app.task_class('tap/file_task')
|
|
528
|
+
# t_class # => Tap::FileTask
|
|
529
|
+
#
|
|
530
|
+
# app.map = {"mapped-task" => "Tap::FileTask"}
|
|
531
|
+
# t_class = app.task_class('mapped-task-1.0')
|
|
532
|
+
# t_class # => Tap::FileTask
|
|
533
|
+
#
|
|
534
|
+
# Notes:
|
|
535
|
+
# - The task class will be auto-loaded using Dependencies, if needed.
|
|
536
|
+
# - A LookupError is raised if the task class cannot be found.
|
|
537
|
+
def task_class(task_name)
|
|
538
|
+
lookup_const task_class_name(task_name) do
|
|
304
539
|
raise LookupError.new("unknown task '#{task_name}'")
|
|
305
540
|
end
|
|
306
541
|
end
|
|
307
542
|
|
|
308
|
-
# Returns the class name of the specified task. If
|
|
309
|
-
# is
|
|
310
|
-
# descriptor, or the class name as specified in map
|
|
543
|
+
# Returns the class name of the specified task. If the task
|
|
544
|
+
# descriptor is a string, the class name is the de-versioned,
|
|
545
|
+
# descriptor, or the class name as specified in map by the
|
|
546
|
+
# de-versioned descriptor.
|
|
547
|
+
#
|
|
548
|
+
# app.map = {"mapped-task" => "Tap::FileTask"}
|
|
549
|
+
# app.task_class_name('some/task_class') # => "some/task_class"
|
|
550
|
+
# app.task_class_name('mapped-task-1.0') # => "Tap::FileTask"
|
|
551
|
+
#
|
|
552
|
+
# If td is a type of Tap::Task::Base, then task_class_name
|
|
553
|
+
# returns td.class.to_s
|
|
554
|
+
#
|
|
555
|
+
# t1 = Task.new
|
|
556
|
+
# app.task_class_name(t1) # => "Tap::Task"
|
|
557
|
+
#
|
|
558
|
+
# t2 = Object.new.extend Tap::Task::Base
|
|
559
|
+
# app.task_class_name(t2) # => "Object"
|
|
311
560
|
#
|
|
312
|
-
# t = Task.new
|
|
313
|
-
# app.map = {"mapped-task" => "AnotherTask"}
|
|
314
|
-
# app.task_class_name(t) # => "Tap::Task"
|
|
315
|
-
# app.task_class_name('mapped-task-1.0') # => "AnotherTask"
|
|
316
561
|
def task_class_name(td)
|
|
317
562
|
case td
|
|
318
563
|
when Tap::Task::Base then td.class.to_s
|
|
319
564
|
else
|
|
320
565
|
# de-version and resolve using map
|
|
321
566
|
name, version = deversion(td.to_s)
|
|
322
|
-
map.has_key?(name) ? map[name] : name
|
|
567
|
+
map.has_key?(name) ? map[name].to_s : name
|
|
323
568
|
end
|
|
324
569
|
end
|
|
325
570
|
|
|
326
|
-
#
|
|
327
|
-
#
|
|
328
|
-
#
|
|
571
|
+
# Iteratively passes the block the configuration templates for the specified file.
|
|
572
|
+
# Ultimately these templates specify configurations for tasks, as well batched tasks,
|
|
573
|
+
# linked to to self. If no block is specified, each_config_template collects the
|
|
574
|
+
# templates and returns them as an array.
|
|
329
575
|
#
|
|
330
|
-
#
|
|
331
|
-
#
|
|
576
|
+
# To make templates, the contents of the file are processed using ERB, then loaded
|
|
577
|
+
# as YAML. ERB for the config files is evaluated in a binding that contains
|
|
578
|
+
# references to self (app) and the input filepath.
|
|
332
579
|
#
|
|
333
|
-
# #
|
|
334
|
-
# #
|
|
335
|
-
# # - key: <%= 1 + 1 %>}
|
|
336
|
-
# app.config_templates("array_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
|
|
580
|
+
# # [simple.yml]
|
|
581
|
+
# # key: value
|
|
337
582
|
#
|
|
338
|
-
#
|
|
339
|
-
#
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
583
|
+
# app.each_config_template("simple.yml") # => [{"key" => "value"}]
|
|
584
|
+
#
|
|
585
|
+
# # [erb.yml]
|
|
586
|
+
# # app: <%= app.object_id %>
|
|
587
|
+
# # filepath: <%= filepath %>
|
|
588
|
+
#
|
|
589
|
+
# app.each_config_template("erb.yml") # => [{"app" => app.object_id, "filepath" => "erb.yml"}]
|
|
590
|
+
#
|
|
591
|
+
# Batched tasks can be specified by providing an array of hashes.
|
|
592
|
+
#
|
|
593
|
+
# # [batched_with_erb.yml]
|
|
594
|
+
# # - key: <%= 1 %>
|
|
595
|
+
# # - key: <%= 1 + 1 %>
|
|
596
|
+
#
|
|
597
|
+
# app.each_config_template("batched_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
|
|
598
|
+
#
|
|
599
|
+
# If no config templates can be loaded (as when the filepath does not exist, or
|
|
600
|
+
# the file is empty), each_config_template passes the block a single empty template.
|
|
601
|
+
def each_config_template(filepath) # :yields: template
|
|
602
|
+
unless block_given?
|
|
603
|
+
templates = []
|
|
604
|
+
each_config_template(filepath) {|template| templates << template}
|
|
605
|
+
return templates
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
if filepath == nil
|
|
609
|
+
yield({})
|
|
346
610
|
else
|
|
347
|
-
|
|
611
|
+
templates = if !File.exists?(filepath) || File.directory?(filepath)
|
|
612
|
+
nil
|
|
613
|
+
else
|
|
614
|
+
# create the reference to app for templating
|
|
615
|
+
app = self
|
|
616
|
+
input = ERB.new(File.read(filepath)).result(binding)
|
|
617
|
+
YAML.load(input)
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
case templates
|
|
621
|
+
when Array
|
|
622
|
+
templates.each do |template|
|
|
623
|
+
yield(template)
|
|
624
|
+
end
|
|
625
|
+
when Hash
|
|
626
|
+
yield(templates)
|
|
627
|
+
else
|
|
628
|
+
yield({})
|
|
629
|
+
end
|
|
348
630
|
end
|
|
349
|
-
|
|
350
|
-
|
|
631
|
+
end
|
|
632
|
+
|
|
633
|
+
# Returns the configuration filepath for the specified task name,
|
|
634
|
+
# File.join(app['config'], task_name + ".yml"). Returns nil if
|
|
635
|
+
# task_name==nil.
|
|
636
|
+
def config_filepath(task_name)
|
|
637
|
+
task_name == nil ? nil : filepath('config', task_name + ".yml")
|
|
351
638
|
end
|
|
352
639
|
|
|
353
640
|
#
|
|
354
641
|
# Execution methods
|
|
355
642
|
#
|
|
356
|
-
|
|
357
|
-
#
|
|
358
|
-
#
|
|
359
|
-
#
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
# def execute(task)
|
|
366
|
-
# synchronize do
|
|
367
|
-
# #log(:execute, task.to_dir, Logger::DEBUG) if options.debug
|
|
368
|
-
#
|
|
369
|
-
# unless state == State::READY || state == State::RUN
|
|
370
|
-
# raise "cannot execute unless application state is READY or RUN"
|
|
371
|
-
# end
|
|
372
|
-
#
|
|
373
|
-
# if deq = queue.deq(task)
|
|
374
|
-
# task, inputs = deq
|
|
375
|
-
# task.execute(*inputs)
|
|
376
|
-
# end
|
|
377
|
-
# end
|
|
378
|
-
# end
|
|
643
|
+
|
|
644
|
+
# Executes the input Executable with the inputs. Stores the result in
|
|
645
|
+
# aggregator unless an on_complete block is set. Returns the audited
|
|
646
|
+
# result.
|
|
647
|
+
def execute(m, inputs)
|
|
648
|
+
_result = m._execute(*inputs)
|
|
649
|
+
aggregator.store(_result) unless m.on_complete_block
|
|
650
|
+
_result
|
|
651
|
+
end
|
|
379
652
|
|
|
380
|
-
#
|
|
381
|
-
#
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
# in
|
|
390
|
-
#
|
|
391
|
-
#
|
|
392
|
-
#
|
|
393
|
-
#
|
|
394
|
-
# run
|
|
395
|
-
|
|
653
|
+
# Sets state = State::READY unless the app has a run_thread
|
|
654
|
+
# (ie the app is running). Returns self.
|
|
655
|
+
def ready
|
|
656
|
+
synchronize do
|
|
657
|
+
self.state = State::READY if self.run_thread == nil
|
|
658
|
+
self
|
|
659
|
+
end
|
|
660
|
+
end
|
|
661
|
+
|
|
662
|
+
# Runs the methods in the queue in which they were enqued. Run exists when there
|
|
663
|
+
# are no more enqued methods. Run returns self. An app can only run on one thread
|
|
664
|
+
# at a time. If run is called when self is already running, run returns immediately.
|
|
665
|
+
#
|
|
666
|
+
# === The Run Cycle
|
|
667
|
+
# During run, each method is executed sequentially on the current thread unless
|
|
668
|
+
# m.multithread == true. In this case run switches into a multithreaded mode and
|
|
669
|
+
# launches up to n execution threads (where n is options.max_threads or
|
|
670
|
+
# DEFAULT_MAX_THREADS) each of which can run a multithreaded method.
|
|
671
|
+
#
|
|
672
|
+
# These threads will run methods until a non-multithreaded method reaches the top
|
|
673
|
+
# of the queue. At that point, run waits for the multithreaded methods to complete,
|
|
674
|
+
# and then switches back into the sequential mode. Run never executes multithreaded
|
|
675
|
+
# and non-multithreaded methods at the same time.
|
|
676
|
+
#
|
|
677
|
+
# Run checks the state of self before executing a method. If the state is changed
|
|
678
|
+
# to State::STOP, then no more methods will be executed (but currently running methods
|
|
679
|
+
# will continute to completion). If the state is changed to State::TERMINATE then
|
|
680
|
+
# no more methods will be executed and currently running methods will be discontinued
|
|
681
|
+
# as described below.
|
|
682
|
+
#
|
|
683
|
+
# When a series of multithreaded methods are stopped or terminated mid-execution,
|
|
684
|
+
# several methods may be waiting for a free execution thread. These are requeued.
|
|
685
|
+
#
|
|
686
|
+
# === Error Handling and Termination
|
|
687
|
+
# When unhandled errors arise during run, run enters a termination (rescue)
|
|
688
|
+
# routine. During termination a TerminationError is raised in each executing
|
|
689
|
+
# method so that the method exits or begins executing its internal error handling
|
|
690
|
+
# code (perhaps performing rollbacks).
|
|
691
|
+
#
|
|
692
|
+
# The TerminationError is ONLY raised when the method calls Task::Base#check_terminate
|
|
693
|
+
# This method is available to all Task::Base objects, but obviously is NOT available
|
|
694
|
+
# to Executable methods generated by _method. These methods need to check the state
|
|
695
|
+
# of app themselves; otherwise they will continue on to completion even when app
|
|
696
|
+
# is in State::TERMINATE.
|
|
697
|
+
#
|
|
698
|
+
# # this task will loop until app.terminate
|
|
699
|
+
# Task.new {|task| while(true) task.check_terminate end }
|
|
700
|
+
#
|
|
701
|
+
# # this task will NEVER terminate
|
|
702
|
+
# Task.new {|task| while(true) end; task.check_terminate }
|
|
703
|
+
#
|
|
704
|
+
# Additional errors that arise during termination are collected and packaged
|
|
705
|
+
# with the orignal error into a RunError. By default all errors are logged
|
|
706
|
+
# and run exits. If debug? == true, then the RunError will be raised for further
|
|
707
|
+
# handling.
|
|
708
|
+
#
|
|
709
|
+
# Note: the method that caused the original unhandled error is no longer executing
|
|
710
|
+
# when termination begins and thus will not recieve a TerminationError.
|
|
711
|
+
def run
|
|
396
712
|
synchronize do
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
# lookup and enqueue the task, if provided
|
|
400
|
-
specified_task = (td == nil || td.kind_of?(Tap::Task::Base) ? td : task(td))
|
|
401
|
-
queue.enq(specified_task, *inputs) if specified_task
|
|
713
|
+
return self unless self.ready.state == State::READY
|
|
402
714
|
|
|
403
|
-
|
|
715
|
+
self.run_thread = Thread.current
|
|
404
716
|
self.state = State::RUN
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
unless task
|
|
416
|
-
clear_threads
|
|
417
|
-
task, inputs = queue.deq
|
|
418
|
-
end
|
|
419
|
-
|
|
420
|
-
# break -- no executable task was found
|
|
421
|
-
break if task.nil?
|
|
422
|
-
|
|
423
|
-
if thread_queue && task.multithread?
|
|
424
|
-
# TODO: log enqueuing task to thread
|
|
425
|
-
|
|
426
|
-
# generate threads as needed and allowed
|
|
427
|
-
# to execute the threads in the thread queue
|
|
428
|
-
start_thread if threads.size < max_threads
|
|
429
|
-
|
|
430
|
-
# NOTE: the producer-consumer relationship of execution
|
|
431
|
-
# threads and the thread_queue means that tasks will sit
|
|
432
|
-
# waiting until an execution thread opens up. in the most
|
|
433
|
-
# extreme case all executing tasks and all tasks in the
|
|
434
|
-
# task_queue could be the same task, each with different
|
|
435
|
-
# inputs. this deviates from the idea of batch processing,
|
|
436
|
-
# but should be rare and not at all fatal given execute
|
|
437
|
-
# synchronization.
|
|
438
|
-
thread_queue.enq [task, inputs]
|
|
439
|
-
|
|
440
|
-
else
|
|
441
|
-
# TODO: log execute task
|
|
442
|
-
|
|
443
|
-
# wait for threads to complete
|
|
444
|
-
# before executing the main thread
|
|
445
|
-
clear_threads
|
|
446
|
-
task.execute(*inputs)
|
|
447
|
-
end
|
|
448
|
-
end
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
# generate threading variables
|
|
720
|
+
max_threads = options.max_threads || DEFAULT_MAX_THREADS
|
|
721
|
+
self.thread_queue = max_threads > 0 ? Queue.new : nil
|
|
722
|
+
|
|
723
|
+
# TODO: log starting run
|
|
724
|
+
begin
|
|
725
|
+
execution_loop do
|
|
726
|
+
break if block_given? && yield(self)
|
|
449
727
|
|
|
450
|
-
# if
|
|
451
|
-
#
|
|
452
|
-
#
|
|
453
|
-
if
|
|
454
|
-
clear_thread_queue
|
|
728
|
+
# if no tasks were in the queue
|
|
729
|
+
# then clear the threads and
|
|
730
|
+
# check for tasks again
|
|
731
|
+
if queue.empty?
|
|
455
732
|
clear_threads
|
|
733
|
+
# break -- no executable task was found
|
|
734
|
+
break if queue.empty?
|
|
456
735
|
end
|
|
457
736
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
737
|
+
m, inputs = queue.deq
|
|
738
|
+
|
|
739
|
+
if thread_queue && m.multithread
|
|
740
|
+
# TODO: log enqueuing task to thread
|
|
741
|
+
|
|
742
|
+
# generate threads as needed and allowed
|
|
743
|
+
# to execute the threads in the thread queue
|
|
744
|
+
start_thread if threads.size < max_threads
|
|
745
|
+
|
|
746
|
+
# NOTE: the producer-consumer relationship of execution
|
|
747
|
+
# threads and the thread_queue means that tasks will sit
|
|
748
|
+
# waiting until an execution thread opens up. in the most
|
|
749
|
+
# extreme case all executing tasks and all tasks in the
|
|
750
|
+
# task_queue could be the same task, each with different
|
|
751
|
+
# inputs. this deviates from the idea of batch processing,
|
|
752
|
+
# but should be rare and not at all fatal given execute
|
|
753
|
+
# synchronization.
|
|
754
|
+
thread_queue.enq [m, inputs]
|
|
755
|
+
|
|
470
756
|
else
|
|
471
|
-
log
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
757
|
+
# TODO: log execute task
|
|
758
|
+
|
|
759
|
+
# wait for threads to complete
|
|
760
|
+
# before executing the main thread
|
|
761
|
+
clear_threads
|
|
762
|
+
execute(m, inputs)
|
|
476
763
|
end
|
|
477
764
|
end
|
|
478
765
|
|
|
766
|
+
# if the run loop exited due to a STOP state,
|
|
767
|
+
# tasks may still be in the thread queue and/or
|
|
768
|
+
# running. be sure these are cleared
|
|
769
|
+
clear_thread_queue
|
|
770
|
+
clear_threads
|
|
771
|
+
|
|
772
|
+
rescue
|
|
773
|
+
# when an error is generated, be sure to terminate
|
|
774
|
+
# all threads so they can clean up after themselves.
|
|
775
|
+
# clear the thread queue first so no more tasks are
|
|
776
|
+
# executed. collect any errors that arise during
|
|
777
|
+
# termination.
|
|
778
|
+
clear_thread_queue
|
|
779
|
+
errors = [$!] + clear_threads(false)
|
|
780
|
+
errors.delete_if {|error| error.kind_of?(TerminateError) }
|
|
781
|
+
|
|
782
|
+
# handle the errors accordingly
|
|
783
|
+
case
|
|
784
|
+
when debug?
|
|
785
|
+
raise Tap::Support::RunError.new(errors)
|
|
786
|
+
else
|
|
787
|
+
errors.each_with_index do |err, index|
|
|
788
|
+
log("RunError [#{index}] #{err.class}", err.message)
|
|
789
|
+
end
|
|
790
|
+
end
|
|
791
|
+
ensure
|
|
792
|
+
|
|
479
793
|
# reset run variables
|
|
480
794
|
self.thread_queue = nil
|
|
481
|
-
self.state = State::READY
|
|
482
|
-
|
|
483
|
-
# TODO: log run complete
|
|
484
795
|
|
|
485
|
-
|
|
796
|
+
synchronize do
|
|
797
|
+
self.run_thread = nil
|
|
798
|
+
self.state = State::READY
|
|
799
|
+
end
|
|
486
800
|
end
|
|
801
|
+
|
|
802
|
+
# TODO: log run complete
|
|
803
|
+
self
|
|
487
804
|
end
|
|
488
805
|
|
|
489
806
|
# Signals a running application to stop executing tasks in the
|
|
@@ -492,115 +809,153 @@ module Tap
|
|
|
492
809
|
#
|
|
493
810
|
# Does nothing unless state is State::RUN.
|
|
494
811
|
def stop
|
|
495
|
-
|
|
812
|
+
synchronize do
|
|
813
|
+
self.state = State::STOP if self.state == State::RUN
|
|
814
|
+
self
|
|
815
|
+
end
|
|
496
816
|
end
|
|
497
817
|
|
|
498
818
|
# Signals a running application to terminate executing tasks
|
|
499
819
|
# by setting state = State::TERMINATE. When running tasks
|
|
500
|
-
# reach a termination check, the task
|
|
501
|
-
#
|
|
820
|
+
# reach a termination check, the task raises a TerminationError,
|
|
821
|
+
# thus allowing executing tasks to invoke their specific
|
|
502
822
|
# error handling code, perhaps performing rollbacks.
|
|
503
823
|
#
|
|
504
824
|
# Termination checks can be manually specified in a task
|
|
505
|
-
# using the check_terminate method
|
|
506
|
-
# automatically occur before each task execution.
|
|
825
|
+
# using the check_terminate method (see Tap::Task::Base#check_terminate).
|
|
826
|
+
# Termination checks automatically occur before each task execution.
|
|
507
827
|
#
|
|
508
|
-
# Does nothing
|
|
509
|
-
def terminate
|
|
510
|
-
|
|
511
|
-
self.state = State::TERMINATE
|
|
828
|
+
# Does nothing if state == State::READY.
|
|
829
|
+
def terminate
|
|
830
|
+
synchronize do
|
|
831
|
+
self.state = State::TERMINATE unless self.state == State::READY
|
|
832
|
+
self
|
|
512
833
|
end
|
|
513
834
|
end
|
|
514
835
|
|
|
515
836
|
# Returns an information string for the App.
|
|
516
837
|
#
|
|
517
|
-
# App.
|
|
838
|
+
# App.instance.info # => 'state: 0 (READY) queue: 0 thread_queue: 0 threads: 0 results: 0'
|
|
518
839
|
#
|
|
519
840
|
# Provided information:
|
|
520
841
|
#
|
|
521
|
-
# state:: the integer and string values of
|
|
522
|
-
# queue:: the number of
|
|
523
|
-
#
|
|
524
|
-
#
|
|
525
|
-
#
|
|
842
|
+
# state:: the integer and string values of self.state
|
|
843
|
+
# queue:: the number of methods currently in the queue
|
|
844
|
+
# thread_queue:: number of objects in the thread queue, waiting
|
|
845
|
+
# to be run on an execution thread (methods, and
|
|
846
|
+
# perhaps nils to signal threads to clear)
|
|
526
847
|
# threads:: the number of execution threads
|
|
527
|
-
#
|
|
848
|
+
# results:: the total number of results in aggregator
|
|
528
849
|
def info
|
|
529
|
-
|
|
530
|
-
|
|
850
|
+
synchronize do
|
|
851
|
+
"state: #{state} (#{State.state_str(state)}) queue: #{queue.size} thread_queue: #{thread_queue ? thread_queue.size : 0} threads: #{threads.size} results: #{aggregator.size}"
|
|
852
|
+
end
|
|
531
853
|
end
|
|
532
854
|
|
|
533
855
|
#
|
|
534
856
|
# workflow related
|
|
535
857
|
#
|
|
536
858
|
|
|
537
|
-
#
|
|
538
|
-
#
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
# have each task in the batch multithreaded.
|
|
551
|
-
def multithread(*tasks)
|
|
552
|
-
tasks.each do |task|
|
|
553
|
-
task.batch.each {|t| t.multithread = true}
|
|
859
|
+
# Enques the task with the inputs. If the task is batched, then each
|
|
860
|
+
# task in task.batch will be enqued with the inputs. Returns task.
|
|
861
|
+
#
|
|
862
|
+
# An Executable may provided instead of a task.
|
|
863
|
+
def enq(task, *inputs)
|
|
864
|
+
case task
|
|
865
|
+
when Tap::Task::Base
|
|
866
|
+
raise "not assigned to enqueing app: #{task}" unless task.app == self
|
|
867
|
+
task.enq(*inputs)
|
|
868
|
+
when Support::Executable
|
|
869
|
+
queue.enq(task, inputs)
|
|
870
|
+
else
|
|
871
|
+
raise "Not a Task or Executable: #{task}"
|
|
554
872
|
end
|
|
873
|
+
task
|
|
874
|
+
end
|
|
875
|
+
|
|
876
|
+
# Method enque. Enques the specified method from object with the inputs.
|
|
877
|
+
# Returns the enqued method.
|
|
878
|
+
def mq(object, method_name, *inputs)
|
|
879
|
+
m = object._method(method_name)
|
|
880
|
+
enq(m, *inputs)
|
|
555
881
|
end
|
|
556
882
|
|
|
557
883
|
# Sets a sequence workflow pattern for the tasks such that the
|
|
558
884
|
# completion of a task enqueues the next task with it's results.
|
|
559
|
-
#
|
|
560
|
-
#
|
|
561
|
-
#
|
|
562
|
-
|
|
885
|
+
# Batched tasks will have the pattern set for each task in the
|
|
886
|
+
# batch. The current audited results are yielded to the block,
|
|
887
|
+
# if given, before the next task is enqued.
|
|
888
|
+
#
|
|
889
|
+
# Executables may provided as well as tasks.
|
|
890
|
+
def sequence(*tasks) # :yields: _result
|
|
563
891
|
current_task = tasks.shift
|
|
564
|
-
|
|
565
892
|
tasks.each do |next_task|
|
|
566
893
|
# simply pass results from one task to the next.
|
|
567
|
-
on_complete
|
|
568
|
-
|
|
894
|
+
current_task.on_complete do |_result|
|
|
895
|
+
yield(_result) if block_given?
|
|
896
|
+
enq(next_task, _result)
|
|
897
|
+
end
|
|
569
898
|
current_task = next_task
|
|
570
899
|
end
|
|
571
900
|
end
|
|
572
901
|
|
|
573
902
|
# Sets a fork workflow pattern for the tasks such that each of the
|
|
574
903
|
# targets will be enqueued with the results of the source when the
|
|
575
|
-
# source completes.
|
|
576
|
-
#
|
|
577
|
-
#
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
904
|
+
# source completes. Batched tasks will have the pattern set for each
|
|
905
|
+
# task in the batch. The source audited results are yielded to the
|
|
906
|
+
# block, if given, before the targets are enqued.
|
|
907
|
+
#
|
|
908
|
+
# Executables may provided as well as tasks.
|
|
909
|
+
def fork(source, *targets) # :yields: _result
|
|
910
|
+
source.on_complete do |_result|
|
|
911
|
+
targets.each do |target|
|
|
912
|
+
yield(_result) if block_given?
|
|
913
|
+
enq(target, _result)
|
|
585
914
|
end
|
|
586
915
|
end
|
|
587
|
-
targets.each do |target|
|
|
588
|
-
condition(target, &block)
|
|
589
|
-
end if block_given?
|
|
590
916
|
end
|
|
591
917
|
|
|
592
918
|
# Sets a merge workflow pattern for the tasks such that the results
|
|
593
919
|
# of each source will be enqueued to the target when the source
|
|
594
|
-
# completes.
|
|
595
|
-
#
|
|
596
|
-
#
|
|
597
|
-
|
|
920
|
+
# completes. Batched tasks will have the pattern set for each
|
|
921
|
+
# task in the batch. The source audited results are yielded to
|
|
922
|
+
# the block, if given, before the target is enqued.
|
|
923
|
+
#
|
|
924
|
+
# Executables may provided as well as tasks.
|
|
925
|
+
def merge(target, *sources) # :yields: _result
|
|
598
926
|
sources.each do |source|
|
|
599
927
|
# merging can use the existing audit trails... each distinct
|
|
600
928
|
# input is getting sent to one place (the target)
|
|
601
|
-
on_complete
|
|
929
|
+
source.on_complete do |_result|
|
|
930
|
+
yield(_result) if block_given?
|
|
931
|
+
enq(target, _result)
|
|
932
|
+
end
|
|
602
933
|
end
|
|
603
|
-
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
# Returns all aggregated, audited results for the specified tasks.
|
|
937
|
+
# Results are joined into a single array. Arrays of tasks are
|
|
938
|
+
# allowed as inputs. See results.
|
|
939
|
+
def _results(*tasks)
|
|
940
|
+
aggregator.retrieve_all(*tasks.flatten)
|
|
941
|
+
end
|
|
942
|
+
|
|
943
|
+
# Returns all aggregated results for the specified tasks. Results are
|
|
944
|
+
# joined into a single array. Arrays of tasks are allowed as inputs.
|
|
945
|
+
#
|
|
946
|
+
# t1 = Task.new {|task, input| input += 1 }
|
|
947
|
+
# t2 = Task.new {|task, input| input += 10 }
|
|
948
|
+
# t3 = t2.initialize_batch_obj
|
|
949
|
+
#
|
|
950
|
+
# t1.enq(0)
|
|
951
|
+
# t2.enq(1)
|
|
952
|
+
#
|
|
953
|
+
# app.run
|
|
954
|
+
# app.results(t1, t2.batch) # => [1, 11, 11]
|
|
955
|
+
# app.results(t2, t1) # => [11, 1]
|
|
956
|
+
#
|
|
957
|
+
def results(*tasks)
|
|
958
|
+
_results(tasks).collect {|_result| _result._current}
|
|
604
959
|
end
|
|
605
960
|
|
|
606
961
|
protected
|
|
@@ -612,30 +967,34 @@ module Tap
|
|
|
612
967
|
false
|
|
613
968
|
end
|
|
614
969
|
|
|
970
|
+
# Sets the state of the application
|
|
615
971
|
attr_writer :state
|
|
616
|
-
attr_accessor :thread_queue, :threads
|
|
617
972
|
|
|
618
|
-
|
|
973
|
+
# The thread on which run is executing tasks.
|
|
974
|
+
attr_accessor :run_thread
|
|
619
975
|
|
|
976
|
+
# An array containing the execution threads in use by run.
|
|
977
|
+
attr_accessor :threads
|
|
978
|
+
|
|
979
|
+
# A Queue containing multithread tasks waiting to be run
|
|
980
|
+
# on the execution threads. Nil if options.max_threads= 0
|
|
981
|
+
attr_accessor :thread_queue
|
|
982
|
+
|
|
983
|
+
private
|
|
984
|
+
|
|
620
985
|
def execution_loop
|
|
621
986
|
while true
|
|
622
987
|
case state
|
|
623
988
|
when State::STOP
|
|
624
989
|
break
|
|
625
990
|
when State::TERMINATE
|
|
626
|
-
# if an execution thread
|
|
627
|
-
#
|
|
628
|
-
#
|
|
629
|
-
# termination
|
|
630
|
-
# (rescue) code.
|
|
631
|
-
|
|
632
|
-
# BUG - if things just work out that way, terminate can
|
|
633
|
-
# be set from a thread that had an error, then control
|
|
634
|
-
# switches such that this is raised on the main thread.
|
|
635
|
-
# in this case it looks like THIS is the original error.
|
|
991
|
+
# if an execution thread handles the termination error,
|
|
992
|
+
# then the thread may end up here -- terminated but still
|
|
993
|
+
# running. Raise another termination error to enter the
|
|
994
|
+
# termination (rescue) code.
|
|
636
995
|
raise TerminateError.new
|
|
637
996
|
end
|
|
638
|
-
|
|
997
|
+
|
|
639
998
|
yield
|
|
640
999
|
end
|
|
641
1000
|
end
|
|
@@ -657,7 +1016,7 @@ module Tap
|
|
|
657
1016
|
# task being promoted along with it's inputs
|
|
658
1017
|
dequeued.reverse_each do |task, inputs|
|
|
659
1018
|
# TODO: log about not executing
|
|
660
|
-
queue.
|
|
1019
|
+
queue.unshift(task, inputs) unless task.nil?
|
|
661
1020
|
end
|
|
662
1021
|
end
|
|
663
1022
|
|
|
@@ -707,30 +1066,29 @@ module Tap
|
|
|
707
1066
|
|
|
708
1067
|
begin
|
|
709
1068
|
execution_loop do
|
|
710
|
-
|
|
711
|
-
break if
|
|
1069
|
+
m, inputs = thread_queue.deq
|
|
1070
|
+
break if m.nil?
|
|
712
1071
|
|
|
713
1072
|
# TODO: log execute task on thread #
|
|
714
|
-
|
|
1073
|
+
execute(m, inputs)
|
|
715
1074
|
end
|
|
716
1075
|
rescue
|
|
717
|
-
# unless you're already terminating,
|
|
718
1076
|
# an unhandled error should immediately
|
|
719
1077
|
# terminate all threads
|
|
720
|
-
terminate
|
|
721
|
-
Thread.current["error"] = $!
|
|
1078
|
+
terminate
|
|
1079
|
+
Thread.current["error"] = $!
|
|
722
1080
|
end
|
|
723
1081
|
end
|
|
724
1082
|
end
|
|
725
1083
|
end
|
|
726
|
-
|
|
1084
|
+
|
|
727
1085
|
# LookupErrors are raised for errors during dependency lookup
|
|
728
|
-
class LookupError < RuntimeError
|
|
1086
|
+
class LookupError < RuntimeError
|
|
729
1087
|
end
|
|
730
1088
|
|
|
731
1089
|
# TerminateErrors are raised to kill executing tasks when terminate
|
|
732
|
-
# is called on an running App.
|
|
733
|
-
class TerminateError < RuntimeError
|
|
1090
|
+
# is called on an running App. They are handled by the run rescue code.
|
|
1091
|
+
class TerminateError < RuntimeError
|
|
734
1092
|
end
|
|
735
1093
|
end
|
|
736
1094
|
end
|