tap 0.8.0 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. data/Basic Overview +151 -0
  2. data/Command Reference +99 -0
  3. data/History +24 -0
  4. data/MIT-LICENSE +1 -1
  5. data/README +29 -57
  6. data/Rakefile +30 -37
  7. data/Tutorial +243 -191
  8. data/bin/tap +66 -35
  9. data/lib/tap.rb +47 -29
  10. data/lib/tap/app.rb +700 -342
  11. data/lib/tap/{script → cmd}/console.rb +0 -0
  12. data/lib/tap/{script → cmd}/destroy.rb +0 -0
  13. data/lib/tap/{script → cmd}/generate.rb +0 -0
  14. data/lib/tap/cmd/run.rb +156 -0
  15. data/lib/tap/constants.rb +4 -0
  16. data/lib/tap/dump.rb +57 -0
  17. data/lib/tap/env.rb +316 -0
  18. data/lib/tap/file_task.rb +106 -109
  19. data/lib/tap/generator.rb +4 -1
  20. data/lib/tap/generator/generators/command/USAGE +6 -0
  21. data/lib/tap/generator/generators/command/command_generator.rb +17 -0
  22. data/lib/tap/generator/generators/{script/templates/script.erb → command/templates/command.erb} +10 -10
  23. data/lib/tap/generator/generators/config/USAGE +21 -0
  24. data/lib/tap/generator/generators/config/config_generator.rb +17 -7
  25. data/lib/tap/generator/generators/file_task/USAGE +3 -0
  26. data/lib/tap/generator/generators/file_task/file_task_generator.rb +16 -0
  27. data/lib/tap/generator/generators/file_task/templates/file.txt +2 -0
  28. data/lib/tap/generator/generators/file_task/templates/file.yml +3 -0
  29. data/lib/tap/generator/generators/file_task/templates/task.erb +26 -20
  30. data/lib/tap/generator/generators/file_task/templates/test.erb +20 -10
  31. data/lib/tap/generator/generators/generator/generator_generator.rb +1 -1
  32. data/lib/tap/generator/generators/generator/templates/generator.erb +21 -12
  33. data/lib/tap/generator/generators/root/templates/Rakefile +33 -24
  34. data/lib/tap/generator/generators/root/templates/tap.yml +28 -31
  35. data/lib/tap/generator/generators/root/templates/test/tap_test_helper.rb +1 -0
  36. data/lib/tap/generator/generators/task/USAGE +3 -0
  37. data/lib/tap/generator/generators/task/task_generator.rb +18 -5
  38. data/lib/tap/generator/generators/task/templates/task.erb +7 -12
  39. data/lib/tap/generator/generators/task/templates/test.erb +10 -11
  40. data/lib/tap/generator/generators/workflow/templates/task.erb +1 -1
  41. data/lib/tap/generator/generators/workflow/templates/test.erb +1 -1
  42. data/lib/tap/patches/rake/rake_test_loader.rb +8 -0
  43. data/lib/tap/patches/rake/testtask.rb +55 -0
  44. data/lib/tap/patches/ruby19/backtrace_filter.rb +51 -0
  45. data/lib/tap/patches/ruby19/parsedate.rb +16 -0
  46. data/lib/tap/root.rb +172 -67
  47. data/lib/tap/script.rb +70 -336
  48. data/lib/tap/support/aggregator.rb +55 -0
  49. data/lib/tap/support/audit.rb +281 -280
  50. data/lib/tap/support/batchable.rb +59 -0
  51. data/lib/tap/support/class_configuration.rb +279 -0
  52. data/lib/tap/support/configurable.rb +92 -0
  53. data/lib/tap/support/configurable_methods.rb +296 -0
  54. data/lib/tap/support/executable.rb +98 -0
  55. data/lib/tap/support/executable_queue.rb +82 -0
  56. data/lib/tap/support/logger.rb +9 -15
  57. data/lib/tap/support/rake.rb +43 -54
  58. data/lib/tap/support/run_error.rb +32 -13
  59. data/lib/tap/support/shell_utils.rb +47 -0
  60. data/lib/tap/support/tdoc.rb +9 -8
  61. data/lib/tap/support/tdoc/config_attr.rb +40 -16
  62. data/lib/tap/support/validation.rb +77 -0
  63. data/lib/tap/support/versions.rb +36 -36
  64. data/lib/tap/task.rb +276 -482
  65. data/lib/tap/test.rb +20 -261
  66. data/lib/tap/test/env_vars.rb +7 -5
  67. data/lib/tap/test/file_methods.rb +126 -121
  68. data/lib/tap/test/subset_methods.rb +86 -45
  69. data/lib/tap/test/tap_methods.rb +271 -0
  70. data/lib/tap/workflow.rb +174 -46
  71. data/test/app/config/another/task.yml +1 -0
  72. data/test/app/config/erb.yml +2 -1
  73. data/test/app/config/some/task.yml +1 -0
  74. data/test/app/config/template.yml +2 -6
  75. data/test/app_test.rb +1241 -1008
  76. data/test/env/test_configure/recurse_a.yml +2 -0
  77. data/test/env/test_configure/recurse_b.yml +2 -0
  78. data/test/env/test_configure/tap.yml +23 -0
  79. data/test/env/test_load_env_config/dir/tap.yml +3 -0
  80. data/test/env/test_load_env_config/recurse_a.yml +2 -0
  81. data/test/env/test_load_env_config/recurse_b.yml +2 -0
  82. data/test/env/test_load_env_config/tap.yml +3 -0
  83. data/test/env_test.rb +198 -0
  84. data/test/file_task_test.rb +70 -53
  85. data/{lib/tap/generator/generators/package/USAGE → test/root/file.txt} +0 -0
  86. data/test/root_test.rb +621 -454
  87. data/test/script_test.rb +38 -174
  88. data/test/support/aggregator_test.rb +99 -0
  89. data/test/support/audit_test.rb +409 -416
  90. data/test/support/batchable_test.rb +74 -0
  91. data/test/support/{task_configuration_test.rb → class_configuration_test.rb} +106 -47
  92. data/test/{task/config/overriding.yml → support/configurable/config/configured.yml} +0 -0
  93. data/test/support/configurable_test.rb +295 -0
  94. data/test/support/executable_queue_test.rb +103 -0
  95. data/test/support/executable_test.rb +38 -0
  96. data/test/support/logger_test.rb +17 -17
  97. data/test/support/rake_test.rb +4 -2
  98. data/test/support/shell_utils_test.rb +24 -0
  99. data/test/support/tdoc_test.rb +265 -258
  100. data/test/support/validation_test.rb +54 -0
  101. data/test/support/versions_test.rb +38 -38
  102. data/test/tap_test_helper.rb +19 -5
  103. data/test/tap_test_suite.rb +5 -2
  104. data/test/task_base_test.rb +13 -104
  105. data/test/task_syntax_test.rb +300 -0
  106. data/test/task_test.rb +258 -381
  107. data/test/test/env_vars_test.rb +40 -40
  108. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/one.txt +0 -0
  109. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/two.txt +0 -0
  110. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/one.txt +0 -0
  111. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/two.txt +0 -0
  112. data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/one.txt +0 -0
  113. data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/two.txt +0 -0
  114. data/test/test/file_methods/test_assert_files_fails_for_different_content/expected/one.txt +1 -0
  115. data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_different_content}/expected/two.txt +0 -0
  116. data/test/test/file_methods/test_assert_files_fails_for_different_content/input/one.txt +1 -0
  117. data/test/test/file_methods/test_assert_files_fails_for_different_content/input/two.txt +1 -0
  118. data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_missing_expected_file}/expected/one.txt +0 -0
  119. data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/one.txt +1 -0
  120. data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/two.txt +1 -0
  121. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/one.txt +1 -0
  122. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/two.txt +1 -0
  123. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/one.txt +1 -0
  124. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/two.txt +1 -0
  125. data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/one.txt +1 -0
  126. data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/two.txt +1 -0
  127. data/test/test/file_methods_doc/test_sub/expected/one.txt +1 -0
  128. data/test/test/file_methods_doc/test_sub/expected/two.txt +1 -0
  129. data/test/test/file_methods_doc/test_sub/input/one.txt +1 -0
  130. data/test/test/file_methods_doc/test_sub/input/two.txt +1 -0
  131. data/test/test/file_methods_doc_test.rb +29 -0
  132. data/test/test/file_methods_test.rb +214 -143
  133. data/test/test/subset_methods_test.rb +111 -115
  134. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/a.txt +0 -0
  135. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/b.txt +0 -0
  136. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/a.txt +0 -0
  137. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/b.txt +0 -0
  138. data/test/test/tap_methods_test.rb +399 -0
  139. data/test/workflow_test.rb +101 -91
  140. metadata +86 -70
  141. data/lib/tap/generator/generators/package/package_generator.rb +0 -38
  142. data/lib/tap/generator/generators/package/templates/package.erb +0 -186
  143. data/lib/tap/generator/generators/script/USAGE +0 -0
  144. data/lib/tap/generator/generators/script/script_generator.rb +0 -17
  145. data/lib/tap/script/run.rb +0 -154
  146. data/lib/tap/support/batch_queue.rb +0 -162
  147. data/lib/tap/support/combinator.rb +0 -114
  148. data/lib/tap/support/task_configuration.rb +0 -169
  149. data/lib/tap/support/template.rb +0 -81
  150. data/lib/tap/support/templater.rb +0 -155
  151. data/lib/tap/version.rb +0 -4
  152. data/test/app/config/addition_template.yml +0 -6
  153. data/test/app_class_test.rb +0 -33
  154. data/test/check/binding_eval.rb +0 -23
  155. data/test/check/define_method_check.rb +0 -22
  156. data/test/check/dependencies_check.rb +0 -175
  157. data/test/check/inheritance_check.rb +0 -22
  158. data/test/support/batch_queue_test.rb +0 -320
  159. data/test/support/combinator_test.rb +0 -249
  160. data/test/support/template_test.rb +0 -122
  161. data/test/support/templater/erb.txt +0 -2
  162. data/test/support/templater/erb.yml +0 -2
  163. data/test/support/templater/somefile.txt +0 -2
  164. data/test/support/templater_test.rb +0 -192
  165. data/test/task/config/template.yml +0 -4
  166. data/test/task_class_test.rb +0 -170
  167. data/test/task_execute_test.rb +0 -262
  168. data/test/test/file_methods/test_assert_expected/expected/file.txt +0 -1
  169. data/test/test/file_methods/test_assert_expected/expected/folder/file.txt +0 -1
  170. data/test/test/file_methods/test_assert_expected/input/file.txt +0 -1
  171. data/test/test/file_methods/test_assert_expected/input/folder/file.txt +0 -1
  172. data/test/test/file_methods/test_assert_files_exist/input/input_1.txt +0 -0
  173. data/test/test/file_methods/test_assert_files_exist/input/input_2.txt +0 -0
  174. data/test/test/file_methods/test_file_compare/expected/output_1.txt +0 -3
  175. data/test/test/file_methods/test_file_compare/expected/output_2.txt +0 -1
  176. data/test/test/file_methods/test_file_compare/input/input_1.txt +0 -3
  177. data/test/test/file_methods/test_file_compare/input/input_2.txt +0 -3
  178. data/test/test/file_methods/test_infer_glob/expected/file.yml +0 -0
  179. data/test/test/file_methods/test_infer_glob/expected/file_1.txt +0 -0
  180. data/test/test/file_methods/test_infer_glob/expected/file_2.txt +0 -0
  181. data/test/test/file_methods/test_yml_compare/expected/output_1.yml +0 -6
  182. data/test/test/file_methods/test_yml_compare/expected/output_2.yml +0 -6
  183. data/test/test/file_methods/test_yml_compare/input/input_1.yml +0 -4
  184. data/test/test/file_methods/test_yml_compare/input/input_2.yml +0 -4
  185. data/test/test_test.rb +0 -373
data/bin/tap CHANGED
@@ -1,29 +1,60 @@
1
+ #!/usr/local/bin/ruby
1
2
  # usage: tap <command> {options} [args]
2
3
  #
3
4
  # examples:
4
- # tap generate root /path/to/root # generates a root dir
5
+ # tap generate root . # generates a root dir
5
6
  # tap run taskname --option input # runs the 'taskname' task
6
7
  #
7
8
  # help:
8
9
  # tap help # prints this help
9
10
  # tap command --help # prints help for 'command'
11
+ #
10
12
 
11
13
  require File.join( File.dirname(__FILE__), "../lib/tap.rb")
12
- require "tap/script"
13
14
 
15
+ # setup environment
16
+ env = Tap::Env.instance
14
17
  app = Tap::App.instance
15
- script = Tap::Script.instance
16
18
 
19
+ env.logger = app.logger
20
+ if ARGV.delete("-d-")
21
+ env.debug_setup
22
+ end
23
+
24
+ before = nil
25
+ after = nil
26
+
27
+ def handle_error(err)
28
+ case
29
+ when $DEBUG
30
+ puts err.message
31
+ puts
32
+ puts err.backtrace
33
+ when Tap::App.instance.debug? then raise err
34
+ else puts err.message
35
+ end
36
+ end
37
+
38
+ # configure the app to tap.yml if it exists
39
+ default_config_file = File.expand_path( Tap::Env::DEFAULT_CONFIG_FILE )
17
40
  begin
18
- # configure the app to tap.yml if it exists
19
- config_file = Tap::Script.config_filepath(Dir.pwd)
20
- config = Tap::Script.read_config(config_file)
21
- script.configure_app(config)
41
+ env.load_config(default_config_file, app) do |app, config_file, config|
42
+ unless config_file == default_config_file
43
+ env.log(:warn, "ignoring configs: #{config_file} (#{config.keys.join(',')})", Logger::WARN)
44
+ next
45
+ end
46
+
47
+ before = config.delete('before')
48
+ after = config.delete('after')
49
+
50
+ app.reconfigure(config)
51
+ end
22
52
  rescue(Exception)
23
53
  # catch errors and exit gracefully
24
54
  # (errors usu from gem loading errors)
25
55
  puts "Configuration error: #{$!.message}"
26
- puts "Check #{config_file} configurations"
56
+ puts $!.backtrace if $DEBUG
57
+ puts "Check #{default_config_file} configurations"
27
58
  exit(1)
28
59
  end
29
60
 
@@ -36,25 +67,33 @@ end
36
67
  # run before script
37
68
  #
38
69
  begin
39
- eval(script.config.before.to_s)
70
+ eval(before.to_s)
40
71
  rescue
41
72
  puts "Error in before script."
42
- if app.options.debug
43
- raise
44
- else
45
- puts $!.message
46
- exit(1)
47
- end
73
+ handle_error($!)
74
+ exit(1)
48
75
  end
49
76
 
50
77
  begin
51
- available_commands = script.config.scripts
78
+ available_commands = env.commands
52
79
  command = ARGV.shift
53
80
 
54
81
  case command
55
82
  when "--help", "-h", "help", "?", nil
56
83
  # give some help
57
- puts Tap::Script.usage(__FILE__)
84
+ File.open(__FILE__) do |file|
85
+ bang_line = true
86
+ file.each_line do |line|
87
+ if bang_line
88
+ bang_line = false
89
+ next
90
+ end
91
+
92
+ break if line !~ /^#\s?(.*)/
93
+ puts $1
94
+ end
95
+ end
96
+
58
97
  puts
59
98
  puts "available commands:"
60
99
 
@@ -64,36 +103,28 @@ begin
64
103
  print " "
65
104
  puts commands.sort.join("\n ")
66
105
  puts
67
- puts "version #{Tap::VERSION} -- #{Tap::HOMEPAGE}"
68
- else
106
+ puts "version #{Tap::VERSION} -- #{Tap::WEBSITE}"
107
+ else
69
108
  if available_commands.has_key?(command)
70
- # run the script, if it exists
109
+ # run the command, if it exists
71
110
  load available_commands[command]
72
111
  else
73
- puts "Unknown command: '#{command}'"
74
- puts "Type 'tap help' for usage information."
112
+ puts "Unknown command: '#{command}'"
113
+ puts "Type 'tap help' for usage information."
75
114
  end
76
115
  end
77
116
  rescue
78
- if app.options.debug
79
- raise
80
- else
81
- puts $!.message
82
- puts "Type 'tap #{command} --help' for usage information."
83
- end
117
+ handle_error($!)
118
+ puts "Type 'tap #{command} --help' for usage information."
84
119
  end
85
120
 
86
121
  #
87
122
  # run after script
88
123
  #
89
124
  begin
90
- eval(script.config.after.to_s)
125
+ eval(after.to_s)
91
126
  rescue
92
127
  puts "Error in after script."
93
- if app.options.debug
94
- raise
95
- else
96
- puts $!.message
97
- exit(1)
98
- end
128
+ handle_error($!)
129
+ exit(1)
99
130
  end
data/lib/tap.rb CHANGED
@@ -1,44 +1,62 @@
1
- # gem_original_require (set in rubygems) is used here to speed up
2
- # requires when possible... ok since I'm manually activating the
3
- # neeeded gems.
4
1
  require 'rubygems'
5
2
 
6
- gem_original_require 'yaml' # expensive to load
7
- gem_original_require 'logger'
8
- gem_original_require 'ostruct'
9
- gem_original_require 'thread'
10
- gem_original_require 'monitor'
11
- gem_original_require 'erb'
3
+ require 'yaml' # expensive to load
4
+ require 'logger'
5
+ require 'ostruct'
6
+ require 'thread'
7
+ require 'erb'
8
+
9
+ # Apply version-specific patches
10
+ case RUBY_VERSION
11
+ when /^1.9/
12
+ $: << File.dirname(__FILE__) + "/tap/patches/ruby19"
13
+
14
+ # suppresses TDoc warnings
15
+ $DEBUG_RDOC ||= nil
16
+ end
12
17
 
13
18
  # Loading activesupport piecemeal like this cuts the tap load time in half.
14
19
  gem 'activesupport'
15
20
 
16
- gem_original_require 'active_support/core_ext/array/extract_options.rb'
21
+ require 'active_support/core_ext/array/extract_options.rb'
17
22
  class Array #:nodoc:
18
23
  include ActiveSupport::CoreExtensions::Array::ExtractOptions
19
24
  end
20
- gem_original_require 'active_support/core_ext/class.rb'
21
- gem_original_require 'active_support/core_ext/module.rb'
22
- gem_original_require 'active_support/core_ext/symbol.rb'
23
- gem_original_require 'active_support/core_ext/string.rb'
24
- gem_original_require 'active_support/core_ext/blank.rb'
25
- gem_original_require 'active_support/core_ext/hash/keys.rb'
26
- gem_original_require 'active_support/dependencies'
25
+ require 'active_support/core_ext/class.rb'
26
+ require 'active_support/core_ext/module.rb'
27
+ require 'active_support/core_ext/symbol.rb'
28
+ require 'active_support/core_ext/string.rb'
29
+ require 'active_support/core_ext/blank.rb'
30
+ require 'active_support/core_ext/hash/keys.rb'
31
+ require 'active_support/dependencies'
32
+ require 'active_support/clean_logger'
27
33
  class Hash #:nodoc:
28
34
  include ActiveSupport::CoreExtensions::Hash::Keys
29
35
  end
30
36
 
31
37
  $:.unshift File.dirname(__FILE__)
32
38
 
33
- gem_original_require 'tap/support/audit'
34
- gem_original_require 'tap/support/task_configuration'
35
- gem_original_require 'tap/support/logger'
36
- gem_original_require 'tap/support/templater'
37
- gem_original_require 'tap/support/batch_queue'
38
- gem_original_require 'tap/support/run_error'
39
- gem_original_require 'tap/version'
40
- gem_original_require 'tap/root'
41
- gem_original_require 'tap/app'
42
- gem_original_require 'tap/task'
43
- gem_original_require 'tap/workflow'
44
- gem_original_require 'tap/file_task'
39
+ require 'tap/support/aggregator'
40
+ require 'tap/support/audit'
41
+ require 'tap/support/batchable'
42
+ require 'tap/support/class_configuration'
43
+ require 'tap/support/configurable'
44
+ require 'tap/support/configurable_methods'
45
+ require 'tap/support/executable'
46
+ require 'tap/support/executable_queue'
47
+ require 'tap/support/logger'
48
+ require 'tap/support/run_error'
49
+ require 'tap/support/shell_utils'
50
+ require 'tap/support/validation'
51
+ require 'tap/constants'
52
+ require 'tap/env'
53
+ require 'tap/app'
54
+ require 'tap/task'
55
+ require 'tap/file_task'
56
+ require 'tap/workflow'
57
+ require 'tap/dump'
58
+
59
+ # Apply platform-specific patches
60
+ # case RUBY_PLATFORM
61
+ # when 'java'
62
+ # end
@@ -1,63 +1,227 @@
1
1
  module Tap
2
2
 
3
- # == Overview
3
+ # = Overview
4
4
  #
5
5
  # App coordinates the setup and running of tasks, and provides an interface
6
6
  # to the application directory structure. App is convenient for use within
7
7
  # scripts, and provides the basis for the 'tap' command line application.
8
8
  #
9
+ # === Task Setup
10
+ #
9
11
  # All tasks have an App (by default App.instance) which helps initialize the
10
- # task by loading configuration templates from the application directory.
11
- # Task queue commands are passed to app, and tasks access application-
12
- # wide resources like the logger and various options through App.
12
+ # task by loading configuration templates from the config directory. Say
13
+ # we had the following configuration files:
14
+ #
15
+ # [/path/to/app/config/some/task.yml]
16
+ # key: one
17
+ #
18
+ # [/path/to/app/config/another/task.yml]
19
+ # key: two
20
+ #
21
+ # Tasks initialized with the names 'some/task' and 'another/task' will
22
+ # be cofigured by App like this:
23
+ #
24
+ # app = App.instance
25
+ # app.root # => '/path/to/app'
26
+ # app[:config] # => '/path/to/app/config'
27
+ #
28
+ # some_task = Task.new 'some/task'
29
+ # some_task.app # => App.instance
30
+ # some_task.config_file # => '/path/to/app/config/some/task.yml'
31
+ # some_task.config # => {:key => 'one'}
13
32
  #
14
- # task = Task.new {|task, input| input += 1 }
15
- # task.app # => App.instance
16
- # task.enq 1,2,3
33
+ # another_task = Task.new 'another/task'
34
+ # another_task.app # => App.instance
35
+ # another_task.config_file # => '/path/to/app/config/another/task.yml'
36
+ # another_task.config # => {:key => 'two'}
17
37
  #
18
- # task.app.run
38
+ # If app[:config] referenced a different directory then the tasks would be
39
+ # initialized from files relative to that location.
19
40
  #
20
- # task.results.collect do |audit|
21
- # audit._current
22
- # end # => [2,3,4]
41
+ # (see Tap::Root for more details)
23
42
  #
24
43
  # === Running Tasks
25
44
  #
26
- # Tasks are run in the order they are originally queued. Batched tasks are
27
- # all queued at the same time and therefore will execute in succession.
28
- # Multithreaded tasks execute cosynchronously, each on their own thread.
45
+ # Task enque commands are passed to app, and tasks access application-wide
46
+ # resources like the logger and options through App.
47
+ #
48
+ # t1 = Task.new {|task, input| input += 1 }
49
+ # t1.enq 0
50
+ # t1.enq 10
51
+ #
52
+ # app.run
53
+ # app.results(t1) # => [1, 11]
54
+ #
55
+ # When a task completes, app collects its results into a data structure that
56
+ # allows access to them as shown above. This behavior can be modified by
57
+ # setting an on_complete block for the task; on_complete blocks can be used
58
+ # to pass results among tasks, allowing the construction of workflows.
59
+ #
60
+ # # clear the previous results
61
+ # app.aggregator.clear
62
+ #
63
+ # t2 = Task.new {|task, input| input += 10 }
64
+ # t1.on_complete {|_result| t2.enq(_result) }
65
+ #
66
+ # t1.enq 0
67
+ # t1.enq 10
68
+ #
69
+ # app.run
70
+ # app.results(t1) # => []
71
+ # app.results(t2) # => [11, 21]
72
+ #
73
+ # Here t1 has no results because the on_complete block passed them to t2 in
74
+ # a simple sequence.
75
+ #
76
+ # === Running Methods
77
+ #
78
+ # Running a task really consists of calling a method. For tasks, the method is
79
+ # basically the block you provide to Task.new, although execution is mediated by
80
+ # Tap::Task#execute and Tap::Task#process so that the block receives the task
81
+ # as a standard input. In subclasses, the method corresponds to the subclass
82
+ # 'process' method.
83
+ #
84
+ # # the block is called to add one to the input
85
+ # Task.new {|task, input| input += 1 }
86
+ #
87
+ # # same thing, but now in a subclass
88
+ # class AddOne < Tap::Task
89
+ # def process(input) input += 1 end
90
+ # end
91
+ #
92
+ # When tasks are enqued, their executable method is pushed onto the queue along
93
+ # with the inputs for the method. Tasks can be batched such that the executable
94
+ # methods of several tasks are enqued at the same time, allowing you to feed the
95
+ # same inputs to multiple methods at once.
96
+ #
97
+ # t1 = Task.new {|task, input| input += 1 }
98
+ # t2 = Task.new {|task, input| input += 10 }
99
+ # Task.batch(t1, t2) # => [t1, t2]
100
+ #
101
+ # t1.enq 0
102
+ # t2.enq 10
103
+ #
104
+ # app.run
105
+ # app.results(t1) # => [1, 11]
106
+ # app.results(t2) # => [10, 20]
107
+ #
108
+ # App also supports multithreading; multithreaded methods execute cosynchronously,
109
+ # each on their own thread (of course, you need to take care to make each method
110
+ # thread safe).
111
+ #
112
+ # lock = Mutex.new
113
+ # array = []
114
+ # t1 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
115
+ # t2 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
116
+ #
117
+ # t1.multithread = true
118
+ # t1.enq
119
+ # t2.multithread = true
120
+ # t2.enq
121
+ #
122
+ # app.run
123
+ # array.length # => 2
124
+ # array[0] == array[1] # => false
125
+ #
126
+ # Since App is geared towards methods, methods from non-task objects can get
127
+ # hooked into a workflow as needed. The preferred way to do so is to make the
128
+ # non-task objects behave like tasks using Task::Base#initialize. The objects
129
+ # can now be enqued, incorporated into workflows, and batched.
130
+ #
131
+ # array = []
132
+ # Task::Base.initialize(array, :push)
133
+ #
134
+ # array.enq(1)
135
+ # array.enq(2)
29
136
  #
30
- # Workflows can be achieved by setting the condition and on_complete blocks
31
- # for a task. Tasks are skipped until their condition block is met by the
32
- # queued inputs. When a task finishes, it executes its on_complete block,
33
- # thereby allowing results to be dispatched to other tasks.
137
+ # array.empty? # => true
138
+ # app.run
139
+ # array # => [1, 2]
34
140
  #
35
- # In this system, the workflow logic exists among the tasks. App keeps on
36
- # running as long as it finds executable tasks in the queue, or until it
37
- # is stopped or terminated.
141
+ # Lastly, if you can't or don't want to turn your object into a task, Tap defines
142
+ # Object#_method to generate executable objects that can be enqued and
143
+ # incorporated into workflows, although they cannot be batched. The mq
144
+ # (method enq) method generates and enques the method in one step.
38
145
  #
39
- # === Error Handling
146
+ # array = []
147
+ # m = array._method(:push)
148
+ #
149
+ # app.enq(m, 1)
150
+ # app.mq(array, :push, 2)
40
151
  #
41
- # When unhandled errors arise during a run, App enters a termination (rescue)
42
- # routine. During termination a TerminationError is raised in each executing
43
- # task so that the task exits, or begins executing its internal error handling
44
- # code (perhaps performing rollbacks).
152
+ # array.empty? # => true
153
+ # app.run
154
+ # array # => [1, 2]
45
155
  #
46
- # Additional errors that arise during termination are collected and packaged
47
- # with the orignal error into a RunError. By default all errors are logged
48
- # and the run exits. If options.debug == true, then the RunError will be
49
- # raised for further handling.
156
+ # App keeps running as long as it finds methods in the queue, or until it is stopped
157
+ # or terminated.
50
158
  #
51
- # Note: the task that caused the original unhandled error is no longer executing
52
- # when termination begins and thus will not recieve a TerminationError.
159
+ # (see Tap::Support::Executable, Tap::Task, and Tap::Task::Base for more details)
53
160
  #
54
- #--
55
161
  # === Auditing
56
- #++
162
+ #
163
+ # All results generated by methods are audited to track how a given input
164
+ # evolves during a workflow.
165
+ #
166
+ # To illustrate auditing, consider a workflow that uses the 'add_one' method
167
+ # to add one to an input until the result is 3, then adds five more with the
168
+ # 'add_five' method. The final result should always be 8.
169
+ #
170
+ # t1 = Tap::Task.new('add_one') {|task, input| input += 1 }
171
+ # t2 = Tap::Task.new('add_five') {|task, input| input += 5 }
172
+ #
173
+ # t1.on_complete do |_result|
174
+ # # _result is the audit; use the _current method
175
+ # # to get the current value in the audit trail
176
+ #
177
+ # _result._current < 3 ? t1.enq(_result) : t2.enq(_result)
178
+ # end
179
+ #
180
+ # t1.enq(0)
181
+ # t1.enq(1)
182
+ # t1.enq(2)
183
+ #
184
+ # app.run
185
+ # app.results(t2) # => [8,8,8]
186
+ #
187
+ # Although the results are indistinguishable, each achieved the final value
188
+ # through a different series of tasks. With auditing you can see how each
189
+ # input came to the final value of 8:
190
+ #
191
+ # # app.results returns the actual result values
192
+ # # app._results returns the audits for these values
193
+ # app._results(t2).each do |_result|
194
+ # puts "How #{_result._original} became #{_result._current}:"
195
+ # puts _result._to_s
196
+ # puts
197
+ # end
198
+ #
199
+ # Prints:
200
+ #
201
+ # How 2 became 8:
202
+ # o-[] 2
203
+ # o-[add_one] 3
204
+ # o-[add_five] 8
205
+ #
206
+ # How 1 became 8:
207
+ # o-[] 1
208
+ # o-[add_one] 2
209
+ # o-[add_one] 3
210
+ # o-[add_five] 8
211
+ #
212
+ # How 0 became 8:
213
+ # o-[] 0
214
+ # o-[add_one] 1
215
+ # o-[add_one] 2
216
+ # o-[add_one] 3
217
+ # o-[add_five] 8
218
+ #
219
+ # See Tap::Support::Audit for more details.
57
220
  class App < Root
58
221
  include MonitorMixin
59
222
 
60
223
  class << self
224
+ # Sets the current app instance
61
225
  attr_writer :instance
62
226
 
63
227
  # Returns the current instance of App. If no instance has been set,
@@ -65,56 +229,62 @@ module Tap
65
229
  def instance
66
230
  @instance ||= App.new
67
231
  end
68
-
69
- # Runs the specified task and inputs with the current instance.
70
- def run(task, *inputs)
71
- instance.run(task, *inputs)
72
- end
73
-
74
- # Parses the input string as YAML, if the string matches the YAML document
75
- # specifier (ie it begins with "---\s*\n"). Otherwise returns the string.
76
- #
77
- # str = {'key' => 'value'}.to_yaml # => "--- \nkey: value\n"
78
- # Tap::App.parse_yaml(str) # => {'key' => 'value'}
79
- # Tap::App.parse_yaml("str") # => "str"
80
- def parse_yaml(str)
81
- str =~ /^---\s*\n/ ? YAML.load(str) : str
82
- end
83
-
84
- # Read the contents of filepath, templates these using ERB, and loads
85
- # the result as YAML. Returns nil if filepath does not exist.
86
- def read_erb_yaml(filepath)
87
- return nil if !File.exists?(filepath) || File.directory?(filepath)
88
-
89
- input = File.read(filepath)
90
- input = ERB.new(input).result
91
- YAML.load(input)
92
- end
93
232
  end
94
233
 
95
- attr_reader :options, :logger, :queue, :state
234
+ # An OpenStruct containing the application options.
235
+ attr_reader :options
236
+
237
+ # The shared logger.
238
+ attr_reader :logger
239
+
240
+ # The application queue.
241
+ attr_reader :queue
242
+
243
+ # The state of the application (see App::State).
244
+ attr_reader :state
245
+
246
+ # A hash of (task_name, task_class_name) pairs mapping names to
247
+ # classes for instantiating tasks that have a non-default name.
248
+ # See task_class_name for more details.
96
249
  attr_accessor :map
97
250
 
98
- # The constants defining the possible App states.
251
+ # A Tap::Support::Aggregator to collect the results of
252
+ # methods that have no on_complete block.
253
+ attr_reader :aggregator
254
+
255
+ # The constants defining the possible App states.
99
256
  module State
100
257
  READY = 0
101
258
  RUN = 1
102
259
  STOP = 2
103
260
  TERMINATE = 3
261
+
262
+ module_function
263
+
264
+ # Returns the string corresponding to the input state value.
265
+ # Returns nil for unknown states.
266
+ #
267
+ # State.state_str(0) # => 'READY'
268
+ # State.state_str(12) # => nil
269
+ def state_str(state)
270
+ constants.inject(nil) {|str, s| const_get(s) == state ? s.to_s : str}
271
+ end
104
272
  end
105
273
 
106
274
  DEFAULT_MAX_THREADS = 10
107
-
275
+
108
276
  # Creates a new App with the given configuration.
109
277
  # See reconfigure for configuration options.
110
278
  def initialize(config={})
111
279
  super()
112
280
 
113
- @queue = Support::BatchQueue.new
281
+ @state = State::READY
114
282
  @threads = [].extend(MonitorMixin)
115
283
  @thread_queue = nil
116
- @main_thread = nil
117
- @state = State::READY
284
+ @run_thread = nil
285
+
286
+ @queue = Support::ExecutableQueue.new
287
+ @aggregator = Support::Aggregator.new
118
288
 
119
289
  # defaults must be provided for options and logging to ensure
120
290
  # that they will be initialized by reconfigure
@@ -122,6 +292,21 @@ module Tap
122
292
  :options => {}, :logger => {}, :map => {}
123
293
  }.merge(config) )
124
294
  end
295
+
296
+ # Clears the queue and aggregator.
297
+ #def clear(options={})
298
+ # # syncrhonize?
299
+ # ready
300
+ # raise "cannot clear unless state == READY" unless state == State::READY
301
+ #
302
+ # queue.clear
303
+ # aggregator.clear
304
+ #end
305
+
306
+ # True if options.debug or the global variable $DEBUG is true.
307
+ def debug?
308
+ options.debug || $DEBUG ? true : false
309
+ end
125
310
 
126
311
  # Returns the configuration of self.
127
312
  def config
@@ -149,7 +334,9 @@ module Tap
149
334
  # Available configurations:
150
335
  # root:: resets the root directory of self using root=
151
336
  # directories:: resets directory aliases using directories= (note ALL
152
- # aliases are reset. use app[da]= to set a single alias)
337
+ # aliases are reset. use app[dir]= to set a single alias)
338
+ # absolute_paths:: resets absolute path aliases using absolute_paths= (note ALL
339
+ # aliases are reset. use app[dir]= to set a single alias)
153
340
  # options:: resets the application options (note ALL options are reset.
154
341
  # use app.options.opt= to set a single option)
155
342
  # logger:: creates and sets a new logger from the configuration
@@ -159,9 +346,7 @@ module Tap
159
346
  # level:: INFO (1)
160
347
  # datetime_format:: %H:%M:%S
161
348
  #
162
- # Notes:
163
- # - Unknown configurations raise an error.
164
- # - Additional configurations may be added in subclasses using handle_configuration
349
+ # Unknown configurations raise an error.
165
350
  def reconfigure(config={})
166
351
  config = config.symbolize_keys
167
352
 
@@ -212,24 +397,89 @@ module Tap
212
397
  self
213
398
  end
214
399
 
400
+ # Unloads constants loaded by Dependencies, so that they will be reloaded
401
+ # (with any changes made) next time they are called. Returns the unloaded
402
+ # constants.
403
+ def reload
404
+ unloaded = []
405
+
406
+ # echos the behavior of Dependencies.clear,
407
+ # but collects unloaded constants
408
+ Dependencies.loaded.clear
409
+ Dependencies.autoloaded_constants.each do |const|
410
+ Dependencies.remove_constant const
411
+ unloaded << const
412
+ end
413
+ Dependencies.autoloaded_constants.clear
414
+ Dependencies.explicitly_unloadable_constants.each do |const|
415
+ Dependencies.remove_constant const
416
+ unloaded << const
417
+ end
418
+
419
+ unloaded
420
+ end
421
+
422
+ # Looks up the specified constant, dynamically loading via Dependencies
423
+ # if necessary. Returns the const_name if const_name is a Module.
424
+ # Yields to the optional block if the constant cannot be found; otherwise
425
+ # raises a LookupError.
426
+ def lookup_const(const_name)
427
+ return const_name if const_name.kind_of?(Module)
428
+
429
+ begin
430
+ const_name = const_name.camelize
431
+
432
+ case RUBY_VERSION
433
+ when /^1.9/
434
+
435
+ # a check is necessary to maintain the 1.8 behavior
436
+ # of lookup_const in 1.9, where ancestor constants
437
+ # may be returned by a direct evaluation
438
+ const_name.split("::").inject(Object) do |current, const|
439
+ const = const.to_sym
440
+
441
+ current.const_get(const).tap do |c|
442
+ unless current.const_defined?(const, false)
443
+ raise NameError.new("uninitialized constant #{const_name}")
444
+ end
445
+ end
446
+ end
447
+
448
+ else
449
+ Object.module_eval const_name
450
+ end
451
+
452
+ rescue(NameError)
453
+ if block_given?
454
+ yield
455
+ else
456
+ raise LookupError.new("unknown constant: #{const_name}")
457
+ end
458
+ end
459
+ end
460
+
215
461
  #
216
462
  # Logging methods
217
463
  #
218
464
 
219
465
  # Sets the current logger. The logger is extended with Support::Logger to provide
220
- # additional logging capabilities.
466
+ # additional logging capabilities. The logger level is set to Logger::DEBUG if
467
+ # the global variable $DEBUG is true.
221
468
  def logger=(logger)
222
469
  @logger = logger
223
470
  @logger.extend Support::Logger unless @logger.nil?
471
+ @logger.level = Logger::DEBUG if $DEBUG
224
472
  @logger
225
473
  end
226
474
 
227
475
  # Logs the action and message at the input level (default INFO).
228
- # Logging is suppressed if app.options.quiet
476
+ # Logging is suppressed if options.quiet
229
477
  def log(action, msg="", level=Logger::INFO)
230
478
  logger.add(level, msg, action.to_s) unless options.quiet
231
479
  end
232
480
 
481
+ # EXPERIMENTAL
482
+ #
233
483
  # Formatted log. Works like log, but passes the current log format to the
234
484
  # block and uses whatever format the block returns. The format recieves
235
485
  # the following arguments like so:
@@ -238,6 +488,8 @@ module Tap
238
488
  #
239
489
  # By default, if you don't specify a block, flog just chomps a newline off
240
490
  # the format, so your log will be inline.
491
+ #
492
+ # BUG: Not thread safe at the moment.
241
493
  def flog(action="", msg="", level=Logger::INFO) # :yields: format
242
494
  unless options.quiet
243
495
  logger.format_add(level, msg, action) do |format|
@@ -245,245 +497,310 @@ module Tap
245
497
  end
246
498
  end
247
499
  end
248
-
249
- # def monitor(action="beginning", msg="", &block)
250
- # if options.quiet
251
- # yield
252
- # nil
253
- # else
254
- # @monitoring = true
255
- # result = logger.monitor(action, msg, &block)
256
- # @monitoring = false
257
- # result
258
- # end
259
- # end
260
- #
261
- # def tick_monitor(action=nil, msg="", level=Logger::INFO)
262
- # unless options.quiet || !@monitoring
263
- # action.nil? ?
264
- # logger.tick :
265
- # logger.format_add(level, msg, action.to_s) {|format| "\n" + format}
266
- # end
267
- # end
268
- #
269
- # def monitor_stats(hash)
270
- # unless options.quiet || !@monitoring
271
- # logger << "\n"
272
- # logger << hash.stringify_keys.to_yaml
273
- # end
274
- # end
275
500
 
276
501
  #
277
502
  # Task methods
278
503
  #
279
504
 
280
- # Instantiates the specifed task with config (if provided).
505
+ # Instantiates the specifed task with config (if provided). The task
506
+ # class is determined by task_class.
281
507
  #
282
- # app.map = {"mapped-task" => "AnotherTask"}
508
+ # t = app.task('tap/file_task')
509
+ # t.class # => Tap::FileTask
510
+ # t.name # => 'tap/file_task'
283
511
  #
512
+ # app.map = {"mapped-task" => "Tap::FileTask"}
284
513
  # t = app.task('mapped-task-1.0', :key => 'value')
285
- # t.class # => AnotherTask
514
+ # t.class # => Tap::FileTask
286
515
  # t.name # => "mapped-task-1.0"
287
516
  # t.config[:key] # => 'value'
288
517
  #
289
518
  # A new task is instantiated for each call to task; tasks may share the
290
- # same name. The task class is the de-versioned and camelized task name,
291
- # or the class specified in map. The task class will be auto-loaded using
292
- # Dependencies, if needed.
519
+ # same name.
520
+ def task(task_name, config={}, &block)
521
+ task_class(task_name).new(task_name, config, &block)
522
+ end
523
+
524
+ # Looks up the specifed task class. Names are mapped to task classes
525
+ # using task_class_name.
293
526
  #
294
- # A LookupError is raised if the task class cannot be found.
295
- def task(task_name, config=nil)
296
- begin
297
- # lookup the corresponding class and instantiate
298
- constants = task_class_name(task_name).split('::')
299
- task_class = constants.inject(Object) do |klass, const|
300
- klass.const_get(const)
301
- end
302
- task_class.new(task_name, config)
303
- rescue(NameError)
527
+ # t_class = app.task_class('tap/file_task')
528
+ # t_class # => Tap::FileTask
529
+ #
530
+ # app.map = {"mapped-task" => "Tap::FileTask"}
531
+ # t_class = app.task_class('mapped-task-1.0')
532
+ # t_class # => Tap::FileTask
533
+ #
534
+ # Notes:
535
+ # - The task class will be auto-loaded using Dependencies, if needed.
536
+ # - A LookupError is raised if the task class cannot be found.
537
+ def task_class(task_name)
538
+ lookup_const task_class_name(task_name) do
304
539
  raise LookupError.new("unknown task '#{task_name}'")
305
540
  end
306
541
  end
307
542
 
308
- # Returns the class name of the specified task. If a task descriptor
309
- # is given, task_class_name returns the de-versioned, camelized
310
- # descriptor, or the class name as specified in map.
543
+ # Returns the class name of the specified task. If the task
544
+ # descriptor is a string, the class name is the de-versioned,
545
+ # descriptor, or the class name as specified in map by the
546
+ # de-versioned descriptor.
547
+ #
548
+ # app.map = {"mapped-task" => "Tap::FileTask"}
549
+ # app.task_class_name('some/task_class') # => "some/task_class"
550
+ # app.task_class_name('mapped-task-1.0') # => "Tap::FileTask"
551
+ #
552
+ # If td is a type of Tap::Task::Base, then task_class_name
553
+ # returns td.class.to_s
554
+ #
555
+ # t1 = Task.new
556
+ # app.task_class_name(t1) # => "Tap::Task"
557
+ #
558
+ # t2 = Object.new.extend Tap::Task::Base
559
+ # app.task_class_name(t2) # => "Object"
311
560
  #
312
- # t = Task.new
313
- # app.map = {"mapped-task" => "AnotherTask"}
314
- # app.task_class_name(t) # => "Tap::Task"
315
- # app.task_class_name('mapped-task-1.0') # => "AnotherTask"
316
561
  def task_class_name(td)
317
562
  case td
318
563
  when Tap::Task::Base then td.class.to_s
319
564
  else
320
565
  # de-version and resolve using map
321
566
  name, version = deversion(td.to_s)
322
- map.has_key?(name) ? map[name] : name.camelize
567
+ map.has_key?(name) ? map[name].to_s : name
323
568
  end
324
569
  end
325
570
 
326
- # Creates and returns an array of configuration templates for the specified file.
327
- # To make templates, the contents of the file are processed using ERB, then
328
- # loaded as YAML and assembled into an array of hashes.
571
+ # Iteratively passes the block the configuration templates for the specified file.
572
+ # Ultimately these templates specify configurations for tasks, as well batched tasks,
573
+ # linked to to self. If no block is specified, each_config_template collects the
574
+ # templates and returns them as an array.
329
575
  #
330
- # # simple.yml => "key: value"
331
- # app.config_templates("simple.yml") # => [{"key" => "value"}]
576
+ # To make templates, the contents of the file are processed using ERB, then loaded
577
+ # as YAML. ERB for the config files is evaluated in a binding that contains
578
+ # references to self (app) and the input filepath.
332
579
  #
333
- # # array_with_erb.yml => %Q{
334
- # # - key: <%= 1 %>
335
- # # - key: <%= 1 + 1 %>}
336
- # app.config_templates("array_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
580
+ # # [simple.yml]
581
+ # # key: value
337
582
  #
338
- # If no config templates are loaded (as when the filepath does not exist, or the
339
- # file is empty), then the return will be a single empty template -- [{}].
340
- def config_templates(filepath)
341
- config_templates = App.read_erb_yaml(filepath)
342
-
343
- config_templates = case config_templates
344
- when Array then config_templates
345
- when Hash then [config_templates]
583
+ # app.each_config_template("simple.yml") # => [{"key" => "value"}]
584
+ #
585
+ # # [erb.yml]
586
+ # # app: <%= app.object_id %>
587
+ # # filepath: <%= filepath %>
588
+ #
589
+ # app.each_config_template("erb.yml") # => [{"app" => app.object_id, "filepath" => "erb.yml"}]
590
+ #
591
+ # Batched tasks can be specified by providing an array of hashes.
592
+ #
593
+ # # [batched_with_erb.yml]
594
+ # # - key: <%= 1 %>
595
+ # # - key: <%= 1 + 1 %>
596
+ #
597
+ # app.each_config_template("batched_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
598
+ #
599
+ # If no config templates can be loaded (as when the filepath does not exist, or
600
+ # the file is empty), each_config_template passes the block a single empty template.
601
+ def each_config_template(filepath) # :yields: template
602
+ unless block_given?
603
+ templates = []
604
+ each_config_template(filepath) {|template| templates << template}
605
+ return templates
606
+ end
607
+
608
+ if filepath == nil
609
+ yield({})
346
610
  else
347
- return [{}]
611
+ templates = if !File.exists?(filepath) || File.directory?(filepath)
612
+ nil
613
+ else
614
+ # create the reference to app for templating
615
+ app = self
616
+ input = ERB.new(File.read(filepath)).result(binding)
617
+ YAML.load(input)
618
+ end
619
+
620
+ case templates
621
+ when Array
622
+ templates.each do |template|
623
+ yield(template)
624
+ end
625
+ when Hash
626
+ yield(templates)
627
+ else
628
+ yield({})
629
+ end
348
630
  end
349
-
350
- config_templates
631
+ end
632
+
633
+ # Returns the configuration filepath for the specified task name,
634
+ # File.join(app['config'], task_name + ".yml"). Returns nil if
635
+ # task_name==nil.
636
+ def config_filepath(task_name)
637
+ task_name == nil ? nil : filepath('config', task_name + ".yml")
351
638
  end
352
639
 
353
640
  #
354
641
  # Execution methods
355
642
  #
356
-
357
- # Dequeues the task and inputs and executes the task with the current task inputs.
358
- #
359
- # Execute differs from run in several significant ways:
360
- # - Only the provided task will be executed (ie the task batch will not be executed)
361
- # - Execution is on the current thread even if the task is multithreaded.
362
- # - No error handling is performed
363
- #
364
- # Execute can only run if the application state is READY or RUN.
365
- # def execute(task)
366
- # synchronize do
367
- # #log(:execute, task.to_dir, Logger::DEBUG) if options.debug
368
- #
369
- # unless state == State::READY || state == State::RUN
370
- # raise "cannot execute unless application state is READY or RUN"
371
- # end
372
- #
373
- # if deq = queue.deq(task)
374
- # task, inputs = deq
375
- # task.execute(*inputs)
376
- # end
377
- # end
378
- # end
643
+
644
+ # Executes the input Executable with the inputs. Stores the result in
645
+ # aggregator unless an on_complete block is set. Returns the audited
646
+ # result.
647
+ def execute(m, inputs)
648
+ _result = m._execute(*inputs)
649
+ aggregator.store(_result) unless m.on_complete_block
650
+ _result
651
+ end
379
652
 
380
- # Enqueues the task and inputs (if provided), then begins the run cycle.
381
- #
382
- # During the run cycle, the app will iterate through the queue and execute
383
- # tasks. Tasks will be executed ONLY if task.executable? returns true with
384
- # the currently enqueued inputs. Tasks are executed on their own thread if
385
- # task.multithread? is true.
386
- #
387
- # As a result of these rules, the queue may still contain unexecuted tasks
388
- # a the end of run. These tasks require additional inputs or conditions
389
- # in order to execute. Execution errors are handled as described above.
390
- #
391
- # Tasks may be specified as a task descriptor (ex: "sample/task") or provided
392
- # directly. Task descriptors are resolved using task.
393
- #
394
- # run returns the specified task on completion.
395
- def run(td=nil, *inputs)
653
+ # Sets state = State::READY unless the app has a run_thread
654
+ # (ie the app is running). Returns self.
655
+ def ready
656
+ synchronize do
657
+ self.state = State::READY if self.run_thread == nil
658
+ self
659
+ end
660
+ end
661
+
662
+ # Runs the methods in the queue in which they were enqued. Run exists when there
663
+ # are no more enqued methods. Run returns self. An app can only run on one thread
664
+ # at a time. If run is called when self is already running, run returns immediately.
665
+ #
666
+ # === The Run Cycle
667
+ # During run, each method is executed sequentially on the current thread unless
668
+ # m.multithread == true. In this case run switches into a multithreaded mode and
669
+ # launches up to n execution threads (where n is options.max_threads or
670
+ # DEFAULT_MAX_THREADS) each of which can run a multithreaded method.
671
+ #
672
+ # These threads will run methods until a non-multithreaded method reaches the top
673
+ # of the queue. At that point, run waits for the multithreaded methods to complete,
674
+ # and then switches back into the sequential mode. Run never executes multithreaded
675
+ # and non-multithreaded methods at the same time.
676
+ #
677
+ # Run checks the state of self before executing a method. If the state is changed
678
+ # to State::STOP, then no more methods will be executed (but currently running methods
679
+ # will continute to completion). If the state is changed to State::TERMINATE then
680
+ # no more methods will be executed and currently running methods will be discontinued
681
+ # as described below.
682
+ #
683
+ # When a series of multithreaded methods are stopped or terminated mid-execution,
684
+ # several methods may be waiting for a free execution thread. These are requeued.
685
+ #
686
+ # === Error Handling and Termination
687
+ # When unhandled errors arise during run, run enters a termination (rescue)
688
+ # routine. During termination a TerminationError is raised in each executing
689
+ # method so that the method exits or begins executing its internal error handling
690
+ # code (perhaps performing rollbacks).
691
+ #
692
+ # The TerminationError is ONLY raised when the method calls Task::Base#check_terminate
693
+ # This method is available to all Task::Base objects, but obviously is NOT available
694
+ # to Executable methods generated by _method. These methods need to check the state
695
+ # of app themselves; otherwise they will continue on to completion even when app
696
+ # is in State::TERMINATE.
697
+ #
698
+ # # this task will loop until app.terminate
699
+ # Task.new {|task| while(true) task.check_terminate end }
700
+ #
701
+ # # this task will NEVER terminate
702
+ # Task.new {|task| while(true) end; task.check_terminate }
703
+ #
704
+ # Additional errors that arise during termination are collected and packaged
705
+ # with the orignal error into a RunError. By default all errors are logged
706
+ # and run exits. If debug? == true, then the RunError will be raised for further
707
+ # handling.
708
+ #
709
+ # Note: the method that caused the original unhandled error is no longer executing
710
+ # when termination begins and thus will not recieve a TerminationError.
711
+ def run
396
712
  synchronize do
397
- # TODO: log starting run
398
-
399
- # lookup and enqueue the task, if provided
400
- specified_task = (td == nil || td.kind_of?(Tap::Task::Base) ? td : task(td))
401
- queue.enq(specified_task, *inputs) if specified_task
713
+ return self unless self.ready.state == State::READY
402
714
 
403
- # generate threading variables
715
+ self.run_thread = Thread.current
404
716
  self.state = State::RUN
405
- max_threads = options.max_threads || DEFAULT_MAX_THREADS
406
- self.thread_queue = max_threads > 0 ? Queue.new : nil
407
-
408
- begin
409
- execution_loop do
410
- task, inputs = queue.deq
411
-
412
- # if no tasks were in the queue
413
- # then clear the threads and
414
- # check for tasks again
415
- unless task
416
- clear_threads
417
- task, inputs = queue.deq
418
- end
419
-
420
- # break -- no executable task was found
421
- break if task.nil?
422
-
423
- if thread_queue && task.multithread?
424
- # TODO: log enqueuing task to thread
425
-
426
- # generate threads as needed and allowed
427
- # to execute the threads in the thread queue
428
- start_thread if threads.size < max_threads
429
-
430
- # NOTE: the producer-consumer relationship of execution
431
- # threads and the thread_queue means that tasks will sit
432
- # waiting until an execution thread opens up. in the most
433
- # extreme case all executing tasks and all tasks in the
434
- # task_queue could be the same task, each with different
435
- # inputs. this deviates from the idea of batch processing,
436
- # but should be rare and not at all fatal given execute
437
- # synchronization.
438
- thread_queue.enq [task, inputs]
439
-
440
- else
441
- # TODO: log execute task
442
-
443
- # wait for threads to complete
444
- # before executing the main thread
445
- clear_threads
446
- task.execute(*inputs)
447
- end
448
- end
717
+ end
718
+
719
+ # generate threading variables
720
+ max_threads = options.max_threads || DEFAULT_MAX_THREADS
721
+ self.thread_queue = max_threads > 0 ? Queue.new : nil
722
+
723
+ # TODO: log starting run
724
+ begin
725
+ execution_loop do
726
+ break if block_given? && yield(self)
449
727
 
450
- # if the run loop exited due to a STOP state,
451
- # tasks may still be in the thread queue and/or
452
- # running. be sure these are cleared
453
- if state == State::STOP
454
- clear_thread_queue
728
+ # if no tasks were in the queue
729
+ # then clear the threads and
730
+ # check for tasks again
731
+ if queue.empty?
455
732
  clear_threads
733
+ # break -- no executable task was found
734
+ break if queue.empty?
456
735
  end
457
736
 
458
- rescue
459
- # when an error is generated, be sure to terminate
460
- # all threads so they can clean up after themselves.
461
- # clear the thread queue first so no more tasks are
462
- # executed. collect any errors that arise during
463
- # termination.
464
- clear_thread_queue
465
- errors = clear_threads(false)
466
-
467
- # handle the errors accordingly
468
- if options.debug
469
- raise Tap::Support::RunError.new($!, errors)
737
+ m, inputs = queue.deq
738
+
739
+ if thread_queue && m.multithread
740
+ # TODO: log enqueuing task to thread
741
+
742
+ # generate threads as needed and allowed
743
+ # to execute the threads in the thread queue
744
+ start_thread if threads.size < max_threads
745
+
746
+ # NOTE: the producer-consumer relationship of execution
747
+ # threads and the thread_queue means that tasks will sit
748
+ # waiting until an execution thread opens up. in the most
749
+ # extreme case all executing tasks and all tasks in the
750
+ # task_queue could be the same task, each with different
751
+ # inputs. this deviates from the idea of batch processing,
752
+ # but should be rare and not at all fatal given execute
753
+ # synchronization.
754
+ thread_queue.enq [m, inputs]
755
+
470
756
  else
471
- log($!.class, $!.message)
472
- errors.each_with_index do |err, index|
473
- next if err.nil?
474
- log("[#{index}] #{err.class}", err.message)
475
- end
757
+ # TODO: log execute task
758
+
759
+ # wait for threads to complete
760
+ # before executing the main thread
761
+ clear_threads
762
+ execute(m, inputs)
476
763
  end
477
764
  end
478
765
 
766
+ # if the run loop exited due to a STOP state,
767
+ # tasks may still be in the thread queue and/or
768
+ # running. be sure these are cleared
769
+ clear_thread_queue
770
+ clear_threads
771
+
772
+ rescue
773
+ # when an error is generated, be sure to terminate
774
+ # all threads so they can clean up after themselves.
775
+ # clear the thread queue first so no more tasks are
776
+ # executed. collect any errors that arise during
777
+ # termination.
778
+ clear_thread_queue
779
+ errors = [$!] + clear_threads(false)
780
+ errors.delete_if {|error| error.kind_of?(TerminateError) }
781
+
782
+ # handle the errors accordingly
783
+ case
784
+ when debug?
785
+ raise Tap::Support::RunError.new(errors)
786
+ else
787
+ errors.each_with_index do |err, index|
788
+ log("RunError [#{index}] #{err.class}", err.message)
789
+ end
790
+ end
791
+ ensure
792
+
479
793
  # reset run variables
480
794
  self.thread_queue = nil
481
- self.state = State::READY
482
-
483
- # TODO: log run complete
484
795
 
485
- specified_task
796
+ synchronize do
797
+ self.run_thread = nil
798
+ self.state = State::READY
799
+ end
486
800
  end
801
+
802
+ # TODO: log run complete
803
+ self
487
804
  end
488
805
 
489
806
  # Signals a running application to stop executing tasks in the
@@ -492,115 +809,153 @@ module Tap
492
809
  #
493
810
  # Does nothing unless state is State::RUN.
494
811
  def stop
495
- self.state = State::STOP if state == State::RUN
812
+ synchronize do
813
+ self.state = State::STOP if self.state == State::RUN
814
+ self
815
+ end
496
816
  end
497
817
 
498
818
  # Signals a running application to terminate executing tasks
499
819
  # by setting state = State::TERMINATE. When running tasks
500
- # reach a termination check, the task will raise a Termination
501
- # error, thus allowing executing tasks to invoke their specific
820
+ # reach a termination check, the task raises a TerminationError,
821
+ # thus allowing executing tasks to invoke their specific
502
822
  # error handling code, perhaps performing rollbacks.
503
823
  #
504
824
  # Termination checks can be manually specified in a task
505
- # using the check_terminate method. Termination checks
506
- # automatically occur before each task execution.
825
+ # using the check_terminate method (see Tap::Task::Base#check_terminate).
826
+ # Termination checks automatically occur before each task execution.
507
827
  #
508
- # Does nothing unless state is State::RUN or State::STOP.
509
- def terminate#(error=TerminateError.new)
510
- if state == State::RUN || state == State::STOP
511
- self.state = State::TERMINATE
828
+ # Does nothing if state == State::READY.
829
+ def terminate
830
+ synchronize do
831
+ self.state = State::TERMINATE unless self.state == State::READY
832
+ self
512
833
  end
513
834
  end
514
835
 
515
836
  # Returns an information string for the App.
516
837
  #
517
- # App.new().info # => 'state: 0 (READY) queue: 0 waiting: 0 (0) threads: 0'
838
+ # App.instance.info # => 'state: 0 (READY) queue: 0 thread_queue: 0 threads: 0 results: 0'
518
839
  #
519
840
  # Provided information:
520
841
  #
521
- # state:: the integer and string values of the application state
522
- # queue:: the number of tasks currently in the queue
523
- # waiting:: the number of executable tasks in the queue, in parenthesis is the
524
- # number of objects in the thread queue (tasks, and perhaps nils to
525
- # signal threads to clear)
842
+ # state:: the integer and string values of self.state
843
+ # queue:: the number of methods currently in the queue
844
+ # thread_queue:: number of objects in the thread queue, waiting
845
+ # to be run on an execution thread (methods, and
846
+ # perhaps nils to signal threads to clear)
526
847
  # threads:: the number of execution threads
527
- #
848
+ # results:: the total number of results in aggregator
528
849
  def info
529
- state_str = State.constants.inject(nil) {|str, s| State.const_get(s) == state ? s : str}
530
- "state: #{state} (#{state_str}) queue: #{queue.size} waiting: #{queue.num_executable} (#{thread_queue ? thread_queue.size : 0}) threads: #{threads.size}"
850
+ synchronize do
851
+ "state: #{state} (#{State.state_str(state)}) queue: #{queue.size} thread_queue: #{thread_queue ? thread_queue.size : 0} threads: #{threads.size} results: #{aggregator.size}"
852
+ end
531
853
  end
532
854
 
533
855
  #
534
856
  # workflow related
535
857
  #
536
858
 
537
- # Sets the condition block for the task. If the task is batched,
538
- # then the condition will be set for each task in the batch.
539
- def condition(task, &block) # :yields: task, audited_inputs
540
- task.batch.each {|t| t.condition(&block)}
541
- end
542
-
543
- # Sets the on_complete block for the task. If the task is batched,
544
- # then on_complete will be set for each task in the batch.
545
- def on_complete(task, &block) # :yields: results
546
- task.batch.each {|t| t.on_complete(&block)}
547
- end
548
-
549
- # Sets multithread=true for each input task. Batched tasks will
550
- # have each task in the batch multithreaded.
551
- def multithread(*tasks)
552
- tasks.each do |task|
553
- task.batch.each {|t| t.multithread = true}
859
+ # Enques the task with the inputs. If the task is batched, then each
860
+ # task in task.batch will be enqued with the inputs. Returns task.
861
+ #
862
+ # An Executable may provided instead of a task.
863
+ def enq(task, *inputs)
864
+ case task
865
+ when Tap::Task::Base
866
+ raise "not assigned to enqueing app: #{task}" unless task.app == self
867
+ task.enq(*inputs)
868
+ when Support::Executable
869
+ queue.enq(task, inputs)
870
+ else
871
+ raise "Not a Task or Executable: #{task}"
554
872
  end
873
+ task
874
+ end
875
+
876
+ # Method enque. Enques the specified method from object with the inputs.
877
+ # Returns the enqued method.
878
+ def mq(object, method_name, *inputs)
879
+ m = object._method(method_name)
880
+ enq(m, *inputs)
555
881
  end
556
882
 
557
883
  # Sets a sequence workflow pattern for the tasks such that the
558
884
  # completion of a task enqueues the next task with it's results.
559
- # The condition block between these tasks will be set to the
560
- # input block, if provided. Batched tasks will have the pattern
561
- # set for each task in the batch.
562
- def sequence(*tasks, &block) # :yields: task, audited_inputs
885
+ # Batched tasks will have the pattern set for each task in the
886
+ # batch. The current audited results are yielded to the block,
887
+ # if given, before the next task is enqued.
888
+ #
889
+ # Executables may provided as well as tasks.
890
+ def sequence(*tasks) # :yields: _result
563
891
  current_task = tasks.shift
564
-
565
892
  tasks.each do |next_task|
566
893
  # simply pass results from one task to the next.
567
- on_complete(current_task) {|results| queue.enq(next_task, *results) }
568
- condition(next_task, &block)
894
+ current_task.on_complete do |_result|
895
+ yield(_result) if block_given?
896
+ enq(next_task, _result)
897
+ end
569
898
  current_task = next_task
570
899
  end
571
900
  end
572
901
 
573
902
  # Sets a fork workflow pattern for the tasks such that each of the
574
903
  # targets will be enqueued with the results of the source when the
575
- # source completes. The condition block for the targets will
576
- # be set to the input block, if provided. Batched tasks will have
577
- # the pattern set for each task in the batch.
578
- def fork(source, *targets, &block) # :yields: task, audited_inputs
579
- on_complete(source) do |results|
580
- targets.each do |target|
581
- # forking requires new audit trails because the same
582
- # inputs are getting sent to multiple tasks
583
- forked_results = results.collect {|result| result._fork }
584
- queue.enq(target, *forked_results)
904
+ # source completes. Batched tasks will have the pattern set for each
905
+ # task in the batch. The source audited results are yielded to the
906
+ # block, if given, before the targets are enqued.
907
+ #
908
+ # Executables may provided as well as tasks.
909
+ def fork(source, *targets) # :yields: _result
910
+ source.on_complete do |_result|
911
+ targets.each do |target|
912
+ yield(_result) if block_given?
913
+ enq(target, _result)
585
914
  end
586
915
  end
587
- targets.each do |target|
588
- condition(target, &block)
589
- end if block_given?
590
916
  end
591
917
 
592
918
  # Sets a merge workflow pattern for the tasks such that the results
593
919
  # of each source will be enqueued to the target when the source
594
- # completes. The condition block for the target will be set to the
595
- # input block, if provided. Batched tasks will have the pattern set
596
- # for each task in the batch.
597
- def merge(target, *sources, &block) # :yields: task, audited_inputs
920
+ # completes. Batched tasks will have the pattern set for each
921
+ # task in the batch. The source audited results are yielded to
922
+ # the block, if given, before the target is enqued.
923
+ #
924
+ # Executables may provided as well as tasks.
925
+ def merge(target, *sources) # :yields: _result
598
926
  sources.each do |source|
599
927
  # merging can use the existing audit trails... each distinct
600
928
  # input is getting sent to one place (the target)
601
- on_complete(source) {|results| queue.enq(target, *results) }
929
+ source.on_complete do |_result|
930
+ yield(_result) if block_given?
931
+ enq(target, _result)
932
+ end
602
933
  end
603
- condition(target, &block) if block_given?
934
+ end
935
+
936
+ # Returns all aggregated, audited results for the specified tasks.
937
+ # Results are joined into a single array. Arrays of tasks are
938
+ # allowed as inputs. See results.
939
+ def _results(*tasks)
940
+ aggregator.retrieve_all(*tasks.flatten)
941
+ end
942
+
943
+ # Returns all aggregated results for the specified tasks. Results are
944
+ # joined into a single array. Arrays of tasks are allowed as inputs.
945
+ #
946
+ # t1 = Task.new {|task, input| input += 1 }
947
+ # t2 = Task.new {|task, input| input += 10 }
948
+ # t3 = t2.initialize_batch_obj
949
+ #
950
+ # t1.enq(0)
951
+ # t2.enq(1)
952
+ #
953
+ # app.run
954
+ # app.results(t1, t2.batch) # => [1, 11, 11]
955
+ # app.results(t2, t1) # => [11, 1]
956
+ #
957
+ def results(*tasks)
958
+ _results(tasks).collect {|_result| _result._current}
604
959
  end
605
960
 
606
961
  protected
@@ -612,30 +967,34 @@ module Tap
612
967
  false
613
968
  end
614
969
 
970
+ # Sets the state of the application
615
971
  attr_writer :state
616
- attr_accessor :thread_queue, :threads
617
972
 
618
- private
973
+ # The thread on which run is executing tasks.
974
+ attr_accessor :run_thread
619
975
 
976
+ # An array containing the execution threads in use by run.
977
+ attr_accessor :threads
978
+
979
+ # A Queue containing multithread tasks waiting to be run
980
+ # on the execution threads. Nil if options.max_threads= 0
981
+ attr_accessor :thread_queue
982
+
983
+ private
984
+
620
985
  def execution_loop
621
986
  while true
622
987
  case state
623
988
  when State::STOP
624
989
  break
625
990
  when State::TERMINATE
626
- # if an execution thread (main or multi) handles the
627
- # termination error, then the thread may end up here --
628
- # terminated but still running. Raise another
629
- # termination error to enter the termination
630
- # (rescue) code.
631
-
632
- # BUG - if things just work out that way, terminate can
633
- # be set from a thread that had an error, then control
634
- # switches such that this is raised on the main thread.
635
- # in this case it looks like THIS is the original error.
991
+ # if an execution thread handles the termination error,
992
+ # then the thread may end up here -- terminated but still
993
+ # running. Raise another termination error to enter the
994
+ # termination (rescue) code.
636
995
  raise TerminateError.new
637
996
  end
638
-
997
+
639
998
  yield
640
999
  end
641
1000
  end
@@ -657,7 +1016,7 @@ module Tap
657
1016
  # task being promoted along with it's inputs
658
1017
  dequeued.reverse_each do |task, inputs|
659
1018
  # TODO: log about not executing
660
- queue.priority_enq(task, *inputs) unless task.nil?
1019
+ queue.unshift(task, inputs) unless task.nil?
661
1020
  end
662
1021
  end
663
1022
 
@@ -707,30 +1066,29 @@ module Tap
707
1066
 
708
1067
  begin
709
1068
  execution_loop do
710
- task, inputs = thread_queue.deq
711
- break if task.nil?
1069
+ m, inputs = thread_queue.deq
1070
+ break if m.nil?
712
1071
 
713
1072
  # TODO: log execute task on thread #
714
- task.execute(*inputs)
1073
+ execute(m, inputs)
715
1074
  end
716
1075
  rescue
717
- # unless you're already terminating,
718
1076
  # an unhandled error should immediately
719
1077
  # terminate all threads
720
- terminate unless state == State::TERMINATE
721
- Thread.current["error"] = $! unless $!.kind_of?(TerminateError)
1078
+ terminate
1079
+ Thread.current["error"] = $!
722
1080
  end
723
1081
  end
724
1082
  end
725
1083
  end
726
-
1084
+
727
1085
  # LookupErrors are raised for errors during dependency lookup
728
- class LookupError < RuntimeError # :nodoc:
1086
+ class LookupError < RuntimeError
729
1087
  end
730
1088
 
731
1089
  # TerminateErrors are raised to kill executing tasks when terminate
732
- # is called on an running App.
733
- class TerminateError < RuntimeError
1090
+ # is called on an running App. They are handled by the run rescue code.
1091
+ class TerminateError < RuntimeError
734
1092
  end
735
1093
  end
736
1094
  end