tap 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. data/Basic Overview +151 -0
  2. data/Command Reference +99 -0
  3. data/History +24 -0
  4. data/MIT-LICENSE +1 -1
  5. data/README +29 -57
  6. data/Rakefile +30 -37
  7. data/Tutorial +243 -191
  8. data/bin/tap +66 -35
  9. data/lib/tap.rb +47 -29
  10. data/lib/tap/app.rb +700 -342
  11. data/lib/tap/{script → cmd}/console.rb +0 -0
  12. data/lib/tap/{script → cmd}/destroy.rb +0 -0
  13. data/lib/tap/{script → cmd}/generate.rb +0 -0
  14. data/lib/tap/cmd/run.rb +156 -0
  15. data/lib/tap/constants.rb +4 -0
  16. data/lib/tap/dump.rb +57 -0
  17. data/lib/tap/env.rb +316 -0
  18. data/lib/tap/file_task.rb +106 -109
  19. data/lib/tap/generator.rb +4 -1
  20. data/lib/tap/generator/generators/command/USAGE +6 -0
  21. data/lib/tap/generator/generators/command/command_generator.rb +17 -0
  22. data/lib/tap/generator/generators/{script/templates/script.erb → command/templates/command.erb} +10 -10
  23. data/lib/tap/generator/generators/config/USAGE +21 -0
  24. data/lib/tap/generator/generators/config/config_generator.rb +17 -7
  25. data/lib/tap/generator/generators/file_task/USAGE +3 -0
  26. data/lib/tap/generator/generators/file_task/file_task_generator.rb +16 -0
  27. data/lib/tap/generator/generators/file_task/templates/file.txt +2 -0
  28. data/lib/tap/generator/generators/file_task/templates/file.yml +3 -0
  29. data/lib/tap/generator/generators/file_task/templates/task.erb +26 -20
  30. data/lib/tap/generator/generators/file_task/templates/test.erb +20 -10
  31. data/lib/tap/generator/generators/generator/generator_generator.rb +1 -1
  32. data/lib/tap/generator/generators/generator/templates/generator.erb +21 -12
  33. data/lib/tap/generator/generators/root/templates/Rakefile +33 -24
  34. data/lib/tap/generator/generators/root/templates/tap.yml +28 -31
  35. data/lib/tap/generator/generators/root/templates/test/tap_test_helper.rb +1 -0
  36. data/lib/tap/generator/generators/task/USAGE +3 -0
  37. data/lib/tap/generator/generators/task/task_generator.rb +18 -5
  38. data/lib/tap/generator/generators/task/templates/task.erb +7 -12
  39. data/lib/tap/generator/generators/task/templates/test.erb +10 -11
  40. data/lib/tap/generator/generators/workflow/templates/task.erb +1 -1
  41. data/lib/tap/generator/generators/workflow/templates/test.erb +1 -1
  42. data/lib/tap/patches/rake/rake_test_loader.rb +8 -0
  43. data/lib/tap/patches/rake/testtask.rb +55 -0
  44. data/lib/tap/patches/ruby19/backtrace_filter.rb +51 -0
  45. data/lib/tap/patches/ruby19/parsedate.rb +16 -0
  46. data/lib/tap/root.rb +172 -67
  47. data/lib/tap/script.rb +70 -336
  48. data/lib/tap/support/aggregator.rb +55 -0
  49. data/lib/tap/support/audit.rb +281 -280
  50. data/lib/tap/support/batchable.rb +59 -0
  51. data/lib/tap/support/class_configuration.rb +279 -0
  52. data/lib/tap/support/configurable.rb +92 -0
  53. data/lib/tap/support/configurable_methods.rb +296 -0
  54. data/lib/tap/support/executable.rb +98 -0
  55. data/lib/tap/support/executable_queue.rb +82 -0
  56. data/lib/tap/support/logger.rb +9 -15
  57. data/lib/tap/support/rake.rb +43 -54
  58. data/lib/tap/support/run_error.rb +32 -13
  59. data/lib/tap/support/shell_utils.rb +47 -0
  60. data/lib/tap/support/tdoc.rb +9 -8
  61. data/lib/tap/support/tdoc/config_attr.rb +40 -16
  62. data/lib/tap/support/validation.rb +77 -0
  63. data/lib/tap/support/versions.rb +36 -36
  64. data/lib/tap/task.rb +276 -482
  65. data/lib/tap/test.rb +20 -261
  66. data/lib/tap/test/env_vars.rb +7 -5
  67. data/lib/tap/test/file_methods.rb +126 -121
  68. data/lib/tap/test/subset_methods.rb +86 -45
  69. data/lib/tap/test/tap_methods.rb +271 -0
  70. data/lib/tap/workflow.rb +174 -46
  71. data/test/app/config/another/task.yml +1 -0
  72. data/test/app/config/erb.yml +2 -1
  73. data/test/app/config/some/task.yml +1 -0
  74. data/test/app/config/template.yml +2 -6
  75. data/test/app_test.rb +1241 -1008
  76. data/test/env/test_configure/recurse_a.yml +2 -0
  77. data/test/env/test_configure/recurse_b.yml +2 -0
  78. data/test/env/test_configure/tap.yml +23 -0
  79. data/test/env/test_load_env_config/dir/tap.yml +3 -0
  80. data/test/env/test_load_env_config/recurse_a.yml +2 -0
  81. data/test/env/test_load_env_config/recurse_b.yml +2 -0
  82. data/test/env/test_load_env_config/tap.yml +3 -0
  83. data/test/env_test.rb +198 -0
  84. data/test/file_task_test.rb +70 -53
  85. data/{lib/tap/generator/generators/package/USAGE → test/root/file.txt} +0 -0
  86. data/test/root_test.rb +621 -454
  87. data/test/script_test.rb +38 -174
  88. data/test/support/aggregator_test.rb +99 -0
  89. data/test/support/audit_test.rb +409 -416
  90. data/test/support/batchable_test.rb +74 -0
  91. data/test/support/{task_configuration_test.rb → class_configuration_test.rb} +106 -47
  92. data/test/{task/config/overriding.yml → support/configurable/config/configured.yml} +0 -0
  93. data/test/support/configurable_test.rb +295 -0
  94. data/test/support/executable_queue_test.rb +103 -0
  95. data/test/support/executable_test.rb +38 -0
  96. data/test/support/logger_test.rb +17 -17
  97. data/test/support/rake_test.rb +4 -2
  98. data/test/support/shell_utils_test.rb +24 -0
  99. data/test/support/tdoc_test.rb +265 -258
  100. data/test/support/validation_test.rb +54 -0
  101. data/test/support/versions_test.rb +38 -38
  102. data/test/tap_test_helper.rb +19 -5
  103. data/test/tap_test_suite.rb +5 -2
  104. data/test/task_base_test.rb +13 -104
  105. data/test/task_syntax_test.rb +300 -0
  106. data/test/task_test.rb +258 -381
  107. data/test/test/env_vars_test.rb +40 -40
  108. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/one.txt +0 -0
  109. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/expected/two.txt +0 -0
  110. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/one.txt +0 -0
  111. data/test/test/file_methods/{test_assert_output_files_equal → test_assert_files}/input/two.txt +0 -0
  112. data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/one.txt +0 -0
  113. data/test/test/{test_file_task_test → file_methods/test_assert_files_can_have_no_expected_files_if_specified}/input/two.txt +0 -0
  114. data/test/test/file_methods/test_assert_files_fails_for_different_content/expected/one.txt +1 -0
  115. data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_different_content}/expected/two.txt +0 -0
  116. data/test/test/file_methods/test_assert_files_fails_for_different_content/input/one.txt +1 -0
  117. data/test/test/file_methods/test_assert_files_fails_for_different_content/input/two.txt +1 -0
  118. data/test/test/{test_file_task_test → file_methods/test_assert_files_fails_for_missing_expected_file}/expected/one.txt +0 -0
  119. data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/one.txt +1 -0
  120. data/test/test/file_methods/test_assert_files_fails_for_missing_expected_file/input/two.txt +1 -0
  121. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/one.txt +1 -0
  122. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/expected/two.txt +1 -0
  123. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/one.txt +1 -0
  124. data/test/test/file_methods/test_assert_files_fails_for_missing_output_file/input/two.txt +1 -0
  125. data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/one.txt +1 -0
  126. data/test/test/file_methods/test_assert_files_fails_for_no_expected_files/input/two.txt +1 -0
  127. data/test/test/file_methods_doc/test_sub/expected/one.txt +1 -0
  128. data/test/test/file_methods_doc/test_sub/expected/two.txt +1 -0
  129. data/test/test/file_methods_doc/test_sub/input/one.txt +1 -0
  130. data/test/test/file_methods_doc/test_sub/input/two.txt +1 -0
  131. data/test/test/file_methods_doc_test.rb +29 -0
  132. data/test/test/file_methods_test.rb +214 -143
  133. data/test/test/subset_methods_test.rb +111 -115
  134. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/a.txt +0 -0
  135. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/expected/task/name/b.txt +0 -0
  136. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/a.txt +0 -0
  137. data/test/test/{test_assert_expected_result_files → tap_methods/test_assert_files}/input/b.txt +0 -0
  138. data/test/test/tap_methods_test.rb +399 -0
  139. data/test/workflow_test.rb +101 -91
  140. metadata +86 -70
  141. data/lib/tap/generator/generators/package/package_generator.rb +0 -38
  142. data/lib/tap/generator/generators/package/templates/package.erb +0 -186
  143. data/lib/tap/generator/generators/script/USAGE +0 -0
  144. data/lib/tap/generator/generators/script/script_generator.rb +0 -17
  145. data/lib/tap/script/run.rb +0 -154
  146. data/lib/tap/support/batch_queue.rb +0 -162
  147. data/lib/tap/support/combinator.rb +0 -114
  148. data/lib/tap/support/task_configuration.rb +0 -169
  149. data/lib/tap/support/template.rb +0 -81
  150. data/lib/tap/support/templater.rb +0 -155
  151. data/lib/tap/version.rb +0 -4
  152. data/test/app/config/addition_template.yml +0 -6
  153. data/test/app_class_test.rb +0 -33
  154. data/test/check/binding_eval.rb +0 -23
  155. data/test/check/define_method_check.rb +0 -22
  156. data/test/check/dependencies_check.rb +0 -175
  157. data/test/check/inheritance_check.rb +0 -22
  158. data/test/support/batch_queue_test.rb +0 -320
  159. data/test/support/combinator_test.rb +0 -249
  160. data/test/support/template_test.rb +0 -122
  161. data/test/support/templater/erb.txt +0 -2
  162. data/test/support/templater/erb.yml +0 -2
  163. data/test/support/templater/somefile.txt +0 -2
  164. data/test/support/templater_test.rb +0 -192
  165. data/test/task/config/template.yml +0 -4
  166. data/test/task_class_test.rb +0 -170
  167. data/test/task_execute_test.rb +0 -262
  168. data/test/test/file_methods/test_assert_expected/expected/file.txt +0 -1
  169. data/test/test/file_methods/test_assert_expected/expected/folder/file.txt +0 -1
  170. data/test/test/file_methods/test_assert_expected/input/file.txt +0 -1
  171. data/test/test/file_methods/test_assert_expected/input/folder/file.txt +0 -1
  172. data/test/test/file_methods/test_assert_files_exist/input/input_1.txt +0 -0
  173. data/test/test/file_methods/test_assert_files_exist/input/input_2.txt +0 -0
  174. data/test/test/file_methods/test_file_compare/expected/output_1.txt +0 -3
  175. data/test/test/file_methods/test_file_compare/expected/output_2.txt +0 -1
  176. data/test/test/file_methods/test_file_compare/input/input_1.txt +0 -3
  177. data/test/test/file_methods/test_file_compare/input/input_2.txt +0 -3
  178. data/test/test/file_methods/test_infer_glob/expected/file.yml +0 -0
  179. data/test/test/file_methods/test_infer_glob/expected/file_1.txt +0 -0
  180. data/test/test/file_methods/test_infer_glob/expected/file_2.txt +0 -0
  181. data/test/test/file_methods/test_yml_compare/expected/output_1.yml +0 -6
  182. data/test/test/file_methods/test_yml_compare/expected/output_2.yml +0 -6
  183. data/test/test/file_methods/test_yml_compare/input/input_1.yml +0 -4
  184. data/test/test/file_methods/test_yml_compare/input/input_2.yml +0 -4
  185. data/test/test_test.rb +0 -373
data/bin/tap CHANGED
@@ -1,29 +1,60 @@
1
+ #!/usr/local/bin/ruby
1
2
  # usage: tap <command> {options} [args]
2
3
  #
3
4
  # examples:
4
- # tap generate root /path/to/root # generates a root dir
5
+ # tap generate root . # generates a root dir
5
6
  # tap run taskname --option input # runs the 'taskname' task
6
7
  #
7
8
  # help:
8
9
  # tap help # prints this help
9
10
  # tap command --help # prints help for 'command'
11
+ #
10
12
 
11
13
  require File.join( File.dirname(__FILE__), "../lib/tap.rb")
12
- require "tap/script"
13
14
 
15
+ # setup environment
16
+ env = Tap::Env.instance
14
17
  app = Tap::App.instance
15
- script = Tap::Script.instance
16
18
 
19
+ env.logger = app.logger
20
+ if ARGV.delete("-d-")
21
+ env.debug_setup
22
+ end
23
+
24
+ before = nil
25
+ after = nil
26
+
27
+ def handle_error(err)
28
+ case
29
+ when $DEBUG
30
+ puts err.message
31
+ puts
32
+ puts err.backtrace
33
+ when Tap::App.instance.debug? then raise err
34
+ else puts err.message
35
+ end
36
+ end
37
+
38
+ # configure the app to tap.yml if it exists
39
+ default_config_file = File.expand_path( Tap::Env::DEFAULT_CONFIG_FILE )
17
40
  begin
18
- # configure the app to tap.yml if it exists
19
- config_file = Tap::Script.config_filepath(Dir.pwd)
20
- config = Tap::Script.read_config(config_file)
21
- script.configure_app(config)
41
+ env.load_config(default_config_file, app) do |app, config_file, config|
42
+ unless config_file == default_config_file
43
+ env.log(:warn, "ignoring configs: #{config_file} (#{config.keys.join(',')})", Logger::WARN)
44
+ next
45
+ end
46
+
47
+ before = config.delete('before')
48
+ after = config.delete('after')
49
+
50
+ app.reconfigure(config)
51
+ end
22
52
  rescue(Exception)
23
53
  # catch errors and exit gracefully
24
54
  # (errors usu from gem loading errors)
25
55
  puts "Configuration error: #{$!.message}"
26
- puts "Check #{config_file} configurations"
56
+ puts $!.backtrace if $DEBUG
57
+ puts "Check #{default_config_file} configurations"
27
58
  exit(1)
28
59
  end
29
60
 
@@ -36,25 +67,33 @@ end
36
67
  # run before script
37
68
  #
38
69
  begin
39
- eval(script.config.before.to_s)
70
+ eval(before.to_s)
40
71
  rescue
41
72
  puts "Error in before script."
42
- if app.options.debug
43
- raise
44
- else
45
- puts $!.message
46
- exit(1)
47
- end
73
+ handle_error($!)
74
+ exit(1)
48
75
  end
49
76
 
50
77
  begin
51
- available_commands = script.config.scripts
78
+ available_commands = env.commands
52
79
  command = ARGV.shift
53
80
 
54
81
  case command
55
82
  when "--help", "-h", "help", "?", nil
56
83
  # give some help
57
- puts Tap::Script.usage(__FILE__)
84
+ File.open(__FILE__) do |file|
85
+ bang_line = true
86
+ file.each_line do |line|
87
+ if bang_line
88
+ bang_line = false
89
+ next
90
+ end
91
+
92
+ break if line !~ /^#\s?(.*)/
93
+ puts $1
94
+ end
95
+ end
96
+
58
97
  puts
59
98
  puts "available commands:"
60
99
 
@@ -64,36 +103,28 @@ begin
64
103
  print " "
65
104
  puts commands.sort.join("\n ")
66
105
  puts
67
- puts "version #{Tap::VERSION} -- #{Tap::HOMEPAGE}"
68
- else
106
+ puts "version #{Tap::VERSION} -- #{Tap::WEBSITE}"
107
+ else
69
108
  if available_commands.has_key?(command)
70
- # run the script, if it exists
109
+ # run the command, if it exists
71
110
  load available_commands[command]
72
111
  else
73
- puts "Unknown command: '#{command}'"
74
- puts "Type 'tap help' for usage information."
112
+ puts "Unknown command: '#{command}'"
113
+ puts "Type 'tap help' for usage information."
75
114
  end
76
115
  end
77
116
  rescue
78
- if app.options.debug
79
- raise
80
- else
81
- puts $!.message
82
- puts "Type 'tap #{command} --help' for usage information."
83
- end
117
+ handle_error($!)
118
+ puts "Type 'tap #{command} --help' for usage information."
84
119
  end
85
120
 
86
121
  #
87
122
  # run after script
88
123
  #
89
124
  begin
90
- eval(script.config.after.to_s)
125
+ eval(after.to_s)
91
126
  rescue
92
127
  puts "Error in after script."
93
- if app.options.debug
94
- raise
95
- else
96
- puts $!.message
97
- exit(1)
98
- end
128
+ handle_error($!)
129
+ exit(1)
99
130
  end
data/lib/tap.rb CHANGED
@@ -1,44 +1,62 @@
1
- # gem_original_require (set in rubygems) is used here to speed up
2
- # requires when possible... ok since I'm manually activating the
3
- # neeeded gems.
4
1
  require 'rubygems'
5
2
 
6
- gem_original_require 'yaml' # expensive to load
7
- gem_original_require 'logger'
8
- gem_original_require 'ostruct'
9
- gem_original_require 'thread'
10
- gem_original_require 'monitor'
11
- gem_original_require 'erb'
3
+ require 'yaml' # expensive to load
4
+ require 'logger'
5
+ require 'ostruct'
6
+ require 'thread'
7
+ require 'erb'
8
+
9
+ # Apply version-specific patches
10
+ case RUBY_VERSION
11
+ when /^1.9/
12
+ $: << File.dirname(__FILE__) + "/tap/patches/ruby19"
13
+
14
+ # suppresses TDoc warnings
15
+ $DEBUG_RDOC ||= nil
16
+ end
12
17
 
13
18
  # Loading activesupport piecemeal like this cuts the tap load time in half.
14
19
  gem 'activesupport'
15
20
 
16
- gem_original_require 'active_support/core_ext/array/extract_options.rb'
21
+ require 'active_support/core_ext/array/extract_options.rb'
17
22
  class Array #:nodoc:
18
23
  include ActiveSupport::CoreExtensions::Array::ExtractOptions
19
24
  end
20
- gem_original_require 'active_support/core_ext/class.rb'
21
- gem_original_require 'active_support/core_ext/module.rb'
22
- gem_original_require 'active_support/core_ext/symbol.rb'
23
- gem_original_require 'active_support/core_ext/string.rb'
24
- gem_original_require 'active_support/core_ext/blank.rb'
25
- gem_original_require 'active_support/core_ext/hash/keys.rb'
26
- gem_original_require 'active_support/dependencies'
25
+ require 'active_support/core_ext/class.rb'
26
+ require 'active_support/core_ext/module.rb'
27
+ require 'active_support/core_ext/symbol.rb'
28
+ require 'active_support/core_ext/string.rb'
29
+ require 'active_support/core_ext/blank.rb'
30
+ require 'active_support/core_ext/hash/keys.rb'
31
+ require 'active_support/dependencies'
32
+ require 'active_support/clean_logger'
27
33
  class Hash #:nodoc:
28
34
  include ActiveSupport::CoreExtensions::Hash::Keys
29
35
  end
30
36
 
31
37
  $:.unshift File.dirname(__FILE__)
32
38
 
33
- gem_original_require 'tap/support/audit'
34
- gem_original_require 'tap/support/task_configuration'
35
- gem_original_require 'tap/support/logger'
36
- gem_original_require 'tap/support/templater'
37
- gem_original_require 'tap/support/batch_queue'
38
- gem_original_require 'tap/support/run_error'
39
- gem_original_require 'tap/version'
40
- gem_original_require 'tap/root'
41
- gem_original_require 'tap/app'
42
- gem_original_require 'tap/task'
43
- gem_original_require 'tap/workflow'
44
- gem_original_require 'tap/file_task'
39
+ require 'tap/support/aggregator'
40
+ require 'tap/support/audit'
41
+ require 'tap/support/batchable'
42
+ require 'tap/support/class_configuration'
43
+ require 'tap/support/configurable'
44
+ require 'tap/support/configurable_methods'
45
+ require 'tap/support/executable'
46
+ require 'tap/support/executable_queue'
47
+ require 'tap/support/logger'
48
+ require 'tap/support/run_error'
49
+ require 'tap/support/shell_utils'
50
+ require 'tap/support/validation'
51
+ require 'tap/constants'
52
+ require 'tap/env'
53
+ require 'tap/app'
54
+ require 'tap/task'
55
+ require 'tap/file_task'
56
+ require 'tap/workflow'
57
+ require 'tap/dump'
58
+
59
+ # Apply platform-specific patches
60
+ # case RUBY_PLATFORM
61
+ # when 'java'
62
+ # end
@@ -1,63 +1,227 @@
1
1
  module Tap
2
2
 
3
- # == Overview
3
+ # = Overview
4
4
  #
5
5
  # App coordinates the setup and running of tasks, and provides an interface
6
6
  # to the application directory structure. App is convenient for use within
7
7
  # scripts, and provides the basis for the 'tap' command line application.
8
8
  #
9
+ # === Task Setup
10
+ #
9
11
  # All tasks have an App (by default App.instance) which helps initialize the
10
- # task by loading configuration templates from the application directory.
11
- # Task queue commands are passed to app, and tasks access application-
12
- # wide resources like the logger and various options through App.
12
+ # task by loading configuration templates from the config directory. Say
13
+ # we had the following configuration files:
14
+ #
15
+ # [/path/to/app/config/some/task.yml]
16
+ # key: one
17
+ #
18
+ # [/path/to/app/config/another/task.yml]
19
+ # key: two
20
+ #
21
+ # Tasks initialized with the names 'some/task' and 'another/task' will
22
+ # be cofigured by App like this:
23
+ #
24
+ # app = App.instance
25
+ # app.root # => '/path/to/app'
26
+ # app[:config] # => '/path/to/app/config'
27
+ #
28
+ # some_task = Task.new 'some/task'
29
+ # some_task.app # => App.instance
30
+ # some_task.config_file # => '/path/to/app/config/some/task.yml'
31
+ # some_task.config # => {:key => 'one'}
13
32
  #
14
- # task = Task.new {|task, input| input += 1 }
15
- # task.app # => App.instance
16
- # task.enq 1,2,3
33
+ # another_task = Task.new 'another/task'
34
+ # another_task.app # => App.instance
35
+ # another_task.config_file # => '/path/to/app/config/another/task.yml'
36
+ # another_task.config # => {:key => 'two'}
17
37
  #
18
- # task.app.run
38
+ # If app[:config] referenced a different directory then the tasks would be
39
+ # initialized from files relative to that location.
19
40
  #
20
- # task.results.collect do |audit|
21
- # audit._current
22
- # end # => [2,3,4]
41
+ # (see Tap::Root for more details)
23
42
  #
24
43
  # === Running Tasks
25
44
  #
26
- # Tasks are run in the order they are originally queued. Batched tasks are
27
- # all queued at the same time and therefore will execute in succession.
28
- # Multithreaded tasks execute cosynchronously, each on their own thread.
45
+ # Task enque commands are passed to app, and tasks access application-wide
46
+ # resources like the logger and options through App.
47
+ #
48
+ # t1 = Task.new {|task, input| input += 1 }
49
+ # t1.enq 0
50
+ # t1.enq 10
51
+ #
52
+ # app.run
53
+ # app.results(t1) # => [1, 11]
54
+ #
55
+ # When a task completes, app collects its results into a data structure that
56
+ # allows access to them as shown above. This behavior can be modified by
57
+ # setting an on_complete block for the task; on_complete blocks can be used
58
+ # to pass results among tasks, allowing the construction of workflows.
59
+ #
60
+ # # clear the previous results
61
+ # app.aggregator.clear
62
+ #
63
+ # t2 = Task.new {|task, input| input += 10 }
64
+ # t1.on_complete {|_result| t2.enq(_result) }
65
+ #
66
+ # t1.enq 0
67
+ # t1.enq 10
68
+ #
69
+ # app.run
70
+ # app.results(t1) # => []
71
+ # app.results(t2) # => [11, 21]
72
+ #
73
+ # Here t1 has no results because the on_complete block passed them to t2 in
74
+ # a simple sequence.
75
+ #
76
+ # === Running Methods
77
+ #
78
+ # Running a task really consists of calling a method. For tasks, the method is
79
+ # basically the block you provide to Task.new, although execution is mediated by
80
+ # Tap::Task#execute and Tap::Task#process so that the block receives the task
81
+ # as a standard input. In subclasses, the method corresponds to the subclass
82
+ # 'process' method.
83
+ #
84
+ # # the block is called to add one to the input
85
+ # Task.new {|task, input| input += 1 }
86
+ #
87
+ # # same thing, but now in a subclass
88
+ # class AddOne < Tap::Task
89
+ # def process(input) input += 1 end
90
+ # end
91
+ #
92
+ # When tasks are enqued, their executable method is pushed onto the queue along
93
+ # with the inputs for the method. Tasks can be batched such that the executable
94
+ # methods of several tasks are enqued at the same time, allowing you to feed the
95
+ # same inputs to multiple methods at once.
96
+ #
97
+ # t1 = Task.new {|task, input| input += 1 }
98
+ # t2 = Task.new {|task, input| input += 10 }
99
+ # Task.batch(t1, t2) # => [t1, t2]
100
+ #
101
+ # t1.enq 0
102
+ # t2.enq 10
103
+ #
104
+ # app.run
105
+ # app.results(t1) # => [1, 11]
106
+ # app.results(t2) # => [10, 20]
107
+ #
108
+ # App also supports multithreading; multithreaded methods execute cosynchronously,
109
+ # each on their own thread (of course, you need to take care to make each method
110
+ # thread safe).
111
+ #
112
+ # lock = Mutex.new
113
+ # array = []
114
+ # t1 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
115
+ # t2 = Task.new {|task| lock.synchronize { array << Thread.current.object_id }; sleep 0.1 }
116
+ #
117
+ # t1.multithread = true
118
+ # t1.enq
119
+ # t2.multithread = true
120
+ # t2.enq
121
+ #
122
+ # app.run
123
+ # array.length # => 2
124
+ # array[0] == array[1] # => false
125
+ #
126
+ # Since App is geared towards methods, methods from non-task objects can get
127
+ # hooked into a workflow as needed. The preferred way to do so is to make the
128
+ # non-task objects behave like tasks using Task::Base#initialize. The objects
129
+ # can now be enqued, incorporated into workflows, and batched.
130
+ #
131
+ # array = []
132
+ # Task::Base.initialize(array, :push)
133
+ #
134
+ # array.enq(1)
135
+ # array.enq(2)
29
136
  #
30
- # Workflows can be achieved by setting the condition and on_complete blocks
31
- # for a task. Tasks are skipped until their condition block is met by the
32
- # queued inputs. When a task finishes, it executes its on_complete block,
33
- # thereby allowing results to be dispatched to other tasks.
137
+ # array.empty? # => true
138
+ # app.run
139
+ # array # => [1, 2]
34
140
  #
35
- # In this system, the workflow logic exists among the tasks. App keeps on
36
- # running as long as it finds executable tasks in the queue, or until it
37
- # is stopped or terminated.
141
+ # Lastly, if you can't or don't want to turn your object into a task, Tap defines
142
+ # Object#_method to generate executable objects that can be enqued and
143
+ # incorporated into workflows, although they cannot be batched. The mq
144
+ # (method enq) method generates and enques the method in one step.
38
145
  #
39
- # === Error Handling
146
+ # array = []
147
+ # m = array._method(:push)
148
+ #
149
+ # app.enq(m, 1)
150
+ # app.mq(array, :push, 2)
40
151
  #
41
- # When unhandled errors arise during a run, App enters a termination (rescue)
42
- # routine. During termination a TerminationError is raised in each executing
43
- # task so that the task exits, or begins executing its internal error handling
44
- # code (perhaps performing rollbacks).
152
+ # array.empty? # => true
153
+ # app.run
154
+ # array # => [1, 2]
45
155
  #
46
- # Additional errors that arise during termination are collected and packaged
47
- # with the orignal error into a RunError. By default all errors are logged
48
- # and the run exits. If options.debug == true, then the RunError will be
49
- # raised for further handling.
156
+ # App keeps running as long as it finds methods in the queue, or until it is stopped
157
+ # or terminated.
50
158
  #
51
- # Note: the task that caused the original unhandled error is no longer executing
52
- # when termination begins and thus will not recieve a TerminationError.
159
+ # (see Tap::Support::Executable, Tap::Task, and Tap::Task::Base for more details)
53
160
  #
54
- #--
55
161
  # === Auditing
56
- #++
162
+ #
163
+ # All results generated by methods are audited to track how a given input
164
+ # evolves during a workflow.
165
+ #
166
+ # To illustrate auditing, consider a workflow that uses the 'add_one' method
167
+ # to add one to an input until the result is 3, then adds five more with the
168
+ # 'add_five' method. The final result should always be 8.
169
+ #
170
+ # t1 = Tap::Task.new('add_one') {|task, input| input += 1 }
171
+ # t2 = Tap::Task.new('add_five') {|task, input| input += 5 }
172
+ #
173
+ # t1.on_complete do |_result|
174
+ # # _result is the audit; use the _current method
175
+ # # to get the current value in the audit trail
176
+ #
177
+ # _result._current < 3 ? t1.enq(_result) : t2.enq(_result)
178
+ # end
179
+ #
180
+ # t1.enq(0)
181
+ # t1.enq(1)
182
+ # t1.enq(2)
183
+ #
184
+ # app.run
185
+ # app.results(t2) # => [8,8,8]
186
+ #
187
+ # Although the results are indistinguishable, each achieved the final value
188
+ # through a different series of tasks. With auditing you can see how each
189
+ # input came to the final value of 8:
190
+ #
191
+ # # app.results returns the actual result values
192
+ # # app._results returns the audits for these values
193
+ # app._results(t2).each do |_result|
194
+ # puts "How #{_result._original} became #{_result._current}:"
195
+ # puts _result._to_s
196
+ # puts
197
+ # end
198
+ #
199
+ # Prints:
200
+ #
201
+ # How 2 became 8:
202
+ # o-[] 2
203
+ # o-[add_one] 3
204
+ # o-[add_five] 8
205
+ #
206
+ # How 1 became 8:
207
+ # o-[] 1
208
+ # o-[add_one] 2
209
+ # o-[add_one] 3
210
+ # o-[add_five] 8
211
+ #
212
+ # How 0 became 8:
213
+ # o-[] 0
214
+ # o-[add_one] 1
215
+ # o-[add_one] 2
216
+ # o-[add_one] 3
217
+ # o-[add_five] 8
218
+ #
219
+ # See Tap::Support::Audit for more details.
57
220
  class App < Root
58
221
  include MonitorMixin
59
222
 
60
223
  class << self
224
+ # Sets the current app instance
61
225
  attr_writer :instance
62
226
 
63
227
  # Returns the current instance of App. If no instance has been set,
@@ -65,56 +229,62 @@ module Tap
65
229
  def instance
66
230
  @instance ||= App.new
67
231
  end
68
-
69
- # Runs the specified task and inputs with the current instance.
70
- def run(task, *inputs)
71
- instance.run(task, *inputs)
72
- end
73
-
74
- # Parses the input string as YAML, if the string matches the YAML document
75
- # specifier (ie it begins with "---\s*\n"). Otherwise returns the string.
76
- #
77
- # str = {'key' => 'value'}.to_yaml # => "--- \nkey: value\n"
78
- # Tap::App.parse_yaml(str) # => {'key' => 'value'}
79
- # Tap::App.parse_yaml("str") # => "str"
80
- def parse_yaml(str)
81
- str =~ /^---\s*\n/ ? YAML.load(str) : str
82
- end
83
-
84
- # Read the contents of filepath, templates these using ERB, and loads
85
- # the result as YAML. Returns nil if filepath does not exist.
86
- def read_erb_yaml(filepath)
87
- return nil if !File.exists?(filepath) || File.directory?(filepath)
88
-
89
- input = File.read(filepath)
90
- input = ERB.new(input).result
91
- YAML.load(input)
92
- end
93
232
  end
94
233
 
95
- attr_reader :options, :logger, :queue, :state
234
+ # An OpenStruct containing the application options.
235
+ attr_reader :options
236
+
237
+ # The shared logger.
238
+ attr_reader :logger
239
+
240
+ # The application queue.
241
+ attr_reader :queue
242
+
243
+ # The state of the application (see App::State).
244
+ attr_reader :state
245
+
246
+ # A hash of (task_name, task_class_name) pairs mapping names to
247
+ # classes for instantiating tasks that have a non-default name.
248
+ # See task_class_name for more details.
96
249
  attr_accessor :map
97
250
 
98
- # The constants defining the possible App states.
251
+ # A Tap::Support::Aggregator to collect the results of
252
+ # methods that have no on_complete block.
253
+ attr_reader :aggregator
254
+
255
+ # The constants defining the possible App states.
99
256
  module State
100
257
  READY = 0
101
258
  RUN = 1
102
259
  STOP = 2
103
260
  TERMINATE = 3
261
+
262
+ module_function
263
+
264
+ # Returns the string corresponding to the input state value.
265
+ # Returns nil for unknown states.
266
+ #
267
+ # State.state_str(0) # => 'READY'
268
+ # State.state_str(12) # => nil
269
+ def state_str(state)
270
+ constants.inject(nil) {|str, s| const_get(s) == state ? s.to_s : str}
271
+ end
104
272
  end
105
273
 
106
274
  DEFAULT_MAX_THREADS = 10
107
-
275
+
108
276
  # Creates a new App with the given configuration.
109
277
  # See reconfigure for configuration options.
110
278
  def initialize(config={})
111
279
  super()
112
280
 
113
- @queue = Support::BatchQueue.new
281
+ @state = State::READY
114
282
  @threads = [].extend(MonitorMixin)
115
283
  @thread_queue = nil
116
- @main_thread = nil
117
- @state = State::READY
284
+ @run_thread = nil
285
+
286
+ @queue = Support::ExecutableQueue.new
287
+ @aggregator = Support::Aggregator.new
118
288
 
119
289
  # defaults must be provided for options and logging to ensure
120
290
  # that they will be initialized by reconfigure
@@ -122,6 +292,21 @@ module Tap
122
292
  :options => {}, :logger => {}, :map => {}
123
293
  }.merge(config) )
124
294
  end
295
+
296
+ # Clears the queue and aggregator.
297
+ #def clear(options={})
298
+ # # syncrhonize?
299
+ # ready
300
+ # raise "cannot clear unless state == READY" unless state == State::READY
301
+ #
302
+ # queue.clear
303
+ # aggregator.clear
304
+ #end
305
+
306
+ # True if options.debug or the global variable $DEBUG is true.
307
+ def debug?
308
+ options.debug || $DEBUG ? true : false
309
+ end
125
310
 
126
311
  # Returns the configuration of self.
127
312
  def config
@@ -149,7 +334,9 @@ module Tap
149
334
  # Available configurations:
150
335
  # root:: resets the root directory of self using root=
151
336
  # directories:: resets directory aliases using directories= (note ALL
152
- # aliases are reset. use app[da]= to set a single alias)
337
+ # aliases are reset. use app[dir]= to set a single alias)
338
+ # absolute_paths:: resets absolute path aliases using absolute_paths= (note ALL
339
+ # aliases are reset. use app[dir]= to set a single alias)
153
340
  # options:: resets the application options (note ALL options are reset.
154
341
  # use app.options.opt= to set a single option)
155
342
  # logger:: creates and sets a new logger from the configuration
@@ -159,9 +346,7 @@ module Tap
159
346
  # level:: INFO (1)
160
347
  # datetime_format:: %H:%M:%S
161
348
  #
162
- # Notes:
163
- # - Unknown configurations raise an error.
164
- # - Additional configurations may be added in subclasses using handle_configuration
349
+ # Unknown configurations raise an error.
165
350
  def reconfigure(config={})
166
351
  config = config.symbolize_keys
167
352
 
@@ -212,24 +397,89 @@ module Tap
212
397
  self
213
398
  end
214
399
 
400
+ # Unloads constants loaded by Dependencies, so that they will be reloaded
401
+ # (with any changes made) next time they are called. Returns the unloaded
402
+ # constants.
403
+ def reload
404
+ unloaded = []
405
+
406
+ # echos the behavior of Dependencies.clear,
407
+ # but collects unloaded constants
408
+ Dependencies.loaded.clear
409
+ Dependencies.autoloaded_constants.each do |const|
410
+ Dependencies.remove_constant const
411
+ unloaded << const
412
+ end
413
+ Dependencies.autoloaded_constants.clear
414
+ Dependencies.explicitly_unloadable_constants.each do |const|
415
+ Dependencies.remove_constant const
416
+ unloaded << const
417
+ end
418
+
419
+ unloaded
420
+ end
421
+
422
+ # Looks up the specified constant, dynamically loading via Dependencies
423
+ # if necessary. Returns the const_name if const_name is a Module.
424
+ # Yields to the optional block if the constant cannot be found; otherwise
425
+ # raises a LookupError.
426
+ def lookup_const(const_name)
427
+ return const_name if const_name.kind_of?(Module)
428
+
429
+ begin
430
+ const_name = const_name.camelize
431
+
432
+ case RUBY_VERSION
433
+ when /^1.9/
434
+
435
+ # a check is necessary to maintain the 1.8 behavior
436
+ # of lookup_const in 1.9, where ancestor constants
437
+ # may be returned by a direct evaluation
438
+ const_name.split("::").inject(Object) do |current, const|
439
+ const = const.to_sym
440
+
441
+ current.const_get(const).tap do |c|
442
+ unless current.const_defined?(const, false)
443
+ raise NameError.new("uninitialized constant #{const_name}")
444
+ end
445
+ end
446
+ end
447
+
448
+ else
449
+ Object.module_eval const_name
450
+ end
451
+
452
+ rescue(NameError)
453
+ if block_given?
454
+ yield
455
+ else
456
+ raise LookupError.new("unknown constant: #{const_name}")
457
+ end
458
+ end
459
+ end
460
+
215
461
  #
216
462
  # Logging methods
217
463
  #
218
464
 
219
465
  # Sets the current logger. The logger is extended with Support::Logger to provide
220
- # additional logging capabilities.
466
+ # additional logging capabilities. The logger level is set to Logger::DEBUG if
467
+ # the global variable $DEBUG is true.
221
468
  def logger=(logger)
222
469
  @logger = logger
223
470
  @logger.extend Support::Logger unless @logger.nil?
471
+ @logger.level = Logger::DEBUG if $DEBUG
224
472
  @logger
225
473
  end
226
474
 
227
475
  # Logs the action and message at the input level (default INFO).
228
- # Logging is suppressed if app.options.quiet
476
+ # Logging is suppressed if options.quiet
229
477
  def log(action, msg="", level=Logger::INFO)
230
478
  logger.add(level, msg, action.to_s) unless options.quiet
231
479
  end
232
480
 
481
+ # EXPERIMENTAL
482
+ #
233
483
  # Formatted log. Works like log, but passes the current log format to the
234
484
  # block and uses whatever format the block returns. The format recieves
235
485
  # the following arguments like so:
@@ -238,6 +488,8 @@ module Tap
238
488
  #
239
489
  # By default, if you don't specify a block, flog just chomps a newline off
240
490
  # the format, so your log will be inline.
491
+ #
492
+ # BUG: Not thread safe at the moment.
241
493
  def flog(action="", msg="", level=Logger::INFO) # :yields: format
242
494
  unless options.quiet
243
495
  logger.format_add(level, msg, action) do |format|
@@ -245,245 +497,310 @@ module Tap
245
497
  end
246
498
  end
247
499
  end
248
-
249
- # def monitor(action="beginning", msg="", &block)
250
- # if options.quiet
251
- # yield
252
- # nil
253
- # else
254
- # @monitoring = true
255
- # result = logger.monitor(action, msg, &block)
256
- # @monitoring = false
257
- # result
258
- # end
259
- # end
260
- #
261
- # def tick_monitor(action=nil, msg="", level=Logger::INFO)
262
- # unless options.quiet || !@monitoring
263
- # action.nil? ?
264
- # logger.tick :
265
- # logger.format_add(level, msg, action.to_s) {|format| "\n" + format}
266
- # end
267
- # end
268
- #
269
- # def monitor_stats(hash)
270
- # unless options.quiet || !@monitoring
271
- # logger << "\n"
272
- # logger << hash.stringify_keys.to_yaml
273
- # end
274
- # end
275
500
 
276
501
  #
277
502
  # Task methods
278
503
  #
279
504
 
280
- # Instantiates the specifed task with config (if provided).
505
+ # Instantiates the specifed task with config (if provided). The task
506
+ # class is determined by task_class.
281
507
  #
282
- # app.map = {"mapped-task" => "AnotherTask"}
508
+ # t = app.task('tap/file_task')
509
+ # t.class # => Tap::FileTask
510
+ # t.name # => 'tap/file_task'
283
511
  #
512
+ # app.map = {"mapped-task" => "Tap::FileTask"}
284
513
  # t = app.task('mapped-task-1.0', :key => 'value')
285
- # t.class # => AnotherTask
514
+ # t.class # => Tap::FileTask
286
515
  # t.name # => "mapped-task-1.0"
287
516
  # t.config[:key] # => 'value'
288
517
  #
289
518
  # A new task is instantiated for each call to task; tasks may share the
290
- # same name. The task class is the de-versioned and camelized task name,
291
- # or the class specified in map. The task class will be auto-loaded using
292
- # Dependencies, if needed.
519
+ # same name.
520
+ def task(task_name, config={}, &block)
521
+ task_class(task_name).new(task_name, config, &block)
522
+ end
523
+
524
+ # Looks up the specifed task class. Names are mapped to task classes
525
+ # using task_class_name.
293
526
  #
294
- # A LookupError is raised if the task class cannot be found.
295
- def task(task_name, config=nil)
296
- begin
297
- # lookup the corresponding class and instantiate
298
- constants = task_class_name(task_name).split('::')
299
- task_class = constants.inject(Object) do |klass, const|
300
- klass.const_get(const)
301
- end
302
- task_class.new(task_name, config)
303
- rescue(NameError)
527
+ # t_class = app.task_class('tap/file_task')
528
+ # t_class # => Tap::FileTask
529
+ #
530
+ # app.map = {"mapped-task" => "Tap::FileTask"}
531
+ # t_class = app.task_class('mapped-task-1.0')
532
+ # t_class # => Tap::FileTask
533
+ #
534
+ # Notes:
535
+ # - The task class will be auto-loaded using Dependencies, if needed.
536
+ # - A LookupError is raised if the task class cannot be found.
537
+ def task_class(task_name)
538
+ lookup_const task_class_name(task_name) do
304
539
  raise LookupError.new("unknown task '#{task_name}'")
305
540
  end
306
541
  end
307
542
 
308
- # Returns the class name of the specified task. If a task descriptor
309
- # is given, task_class_name returns the de-versioned, camelized
310
- # descriptor, or the class name as specified in map.
543
+ # Returns the class name of the specified task. If the task
544
+ # descriptor is a string, the class name is the de-versioned,
545
+ # descriptor, or the class name as specified in map by the
546
+ # de-versioned descriptor.
547
+ #
548
+ # app.map = {"mapped-task" => "Tap::FileTask"}
549
+ # app.task_class_name('some/task_class') # => "some/task_class"
550
+ # app.task_class_name('mapped-task-1.0') # => "Tap::FileTask"
551
+ #
552
+ # If td is a type of Tap::Task::Base, then task_class_name
553
+ # returns td.class.to_s
554
+ #
555
+ # t1 = Task.new
556
+ # app.task_class_name(t1) # => "Tap::Task"
557
+ #
558
+ # t2 = Object.new.extend Tap::Task::Base
559
+ # app.task_class_name(t2) # => "Object"
311
560
  #
312
- # t = Task.new
313
- # app.map = {"mapped-task" => "AnotherTask"}
314
- # app.task_class_name(t) # => "Tap::Task"
315
- # app.task_class_name('mapped-task-1.0') # => "AnotherTask"
316
561
  def task_class_name(td)
317
562
  case td
318
563
  when Tap::Task::Base then td.class.to_s
319
564
  else
320
565
  # de-version and resolve using map
321
566
  name, version = deversion(td.to_s)
322
- map.has_key?(name) ? map[name] : name.camelize
567
+ map.has_key?(name) ? map[name].to_s : name
323
568
  end
324
569
  end
325
570
 
326
- # Creates and returns an array of configuration templates for the specified file.
327
- # To make templates, the contents of the file are processed using ERB, then
328
- # loaded as YAML and assembled into an array of hashes.
571
+ # Iteratively passes the block the configuration templates for the specified file.
572
+ # Ultimately these templates specify configurations for tasks, as well batched tasks,
573
+ # linked to to self. If no block is specified, each_config_template collects the
574
+ # templates and returns them as an array.
329
575
  #
330
- # # simple.yml => "key: value"
331
- # app.config_templates("simple.yml") # => [{"key" => "value"}]
576
+ # To make templates, the contents of the file are processed using ERB, then loaded
577
+ # as YAML. ERB for the config files is evaluated in a binding that contains
578
+ # references to self (app) and the input filepath.
332
579
  #
333
- # # array_with_erb.yml => %Q{
334
- # # - key: <%= 1 %>
335
- # # - key: <%= 1 + 1 %>}
336
- # app.config_templates("array_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
580
+ # # [simple.yml]
581
+ # # key: value
337
582
  #
338
- # If no config templates are loaded (as when the filepath does not exist, or the
339
- # file is empty), then the return will be a single empty template -- [{}].
340
- def config_templates(filepath)
341
- config_templates = App.read_erb_yaml(filepath)
342
-
343
- config_templates = case config_templates
344
- when Array then config_templates
345
- when Hash then [config_templates]
583
+ # app.each_config_template("simple.yml") # => [{"key" => "value"}]
584
+ #
585
+ # # [erb.yml]
586
+ # # app: <%= app.object_id %>
587
+ # # filepath: <%= filepath %>
588
+ #
589
+ # app.each_config_template("erb.yml") # => [{"app" => app.object_id, "filepath" => "erb.yml"}]
590
+ #
591
+ # Batched tasks can be specified by providing an array of hashes.
592
+ #
593
+ # # [batched_with_erb.yml]
594
+ # # - key: <%= 1 %>
595
+ # # - key: <%= 1 + 1 %>
596
+ #
597
+ # app.each_config_template("batched_with_erb.yml") # => [{"key" => 1}, {"key" => 2}]
598
+ #
599
+ # If no config templates can be loaded (as when the filepath does not exist, or
600
+ # the file is empty), each_config_template passes the block a single empty template.
601
+ def each_config_template(filepath) # :yields: template
602
+ unless block_given?
603
+ templates = []
604
+ each_config_template(filepath) {|template| templates << template}
605
+ return templates
606
+ end
607
+
608
+ if filepath == nil
609
+ yield({})
346
610
  else
347
- return [{}]
611
+ templates = if !File.exists?(filepath) || File.directory?(filepath)
612
+ nil
613
+ else
614
+ # create the reference to app for templating
615
+ app = self
616
+ input = ERB.new(File.read(filepath)).result(binding)
617
+ YAML.load(input)
618
+ end
619
+
620
+ case templates
621
+ when Array
622
+ templates.each do |template|
623
+ yield(template)
624
+ end
625
+ when Hash
626
+ yield(templates)
627
+ else
628
+ yield({})
629
+ end
348
630
  end
349
-
350
- config_templates
631
+ end
632
+
633
+ # Returns the configuration filepath for the specified task name,
634
+ # File.join(app['config'], task_name + ".yml"). Returns nil if
635
+ # task_name==nil.
636
+ def config_filepath(task_name)
637
+ task_name == nil ? nil : filepath('config', task_name + ".yml")
351
638
  end
352
639
 
353
640
  #
354
641
  # Execution methods
355
642
  #
356
-
357
- # Dequeues the task and inputs and executes the task with the current task inputs.
358
- #
359
- # Execute differs from run in several significant ways:
360
- # - Only the provided task will be executed (ie the task batch will not be executed)
361
- # - Execution is on the current thread even if the task is multithreaded.
362
- # - No error handling is performed
363
- #
364
- # Execute can only run if the application state is READY or RUN.
365
- # def execute(task)
366
- # synchronize do
367
- # #log(:execute, task.to_dir, Logger::DEBUG) if options.debug
368
- #
369
- # unless state == State::READY || state == State::RUN
370
- # raise "cannot execute unless application state is READY or RUN"
371
- # end
372
- #
373
- # if deq = queue.deq(task)
374
- # task, inputs = deq
375
- # task.execute(*inputs)
376
- # end
377
- # end
378
- # end
643
+
644
+ # Executes the input Executable with the inputs. Stores the result in
645
+ # aggregator unless an on_complete block is set. Returns the audited
646
+ # result.
647
+ def execute(m, inputs)
648
+ _result = m._execute(*inputs)
649
+ aggregator.store(_result) unless m.on_complete_block
650
+ _result
651
+ end
379
652
 
380
- # Enqueues the task and inputs (if provided), then begins the run cycle.
381
- #
382
- # During the run cycle, the app will iterate through the queue and execute
383
- # tasks. Tasks will be executed ONLY if task.executable? returns true with
384
- # the currently enqueued inputs. Tasks are executed on their own thread if
385
- # task.multithread? is true.
386
- #
387
- # As a result of these rules, the queue may still contain unexecuted tasks
388
- # a the end of run. These tasks require additional inputs or conditions
389
- # in order to execute. Execution errors are handled as described above.
390
- #
391
- # Tasks may be specified as a task descriptor (ex: "sample/task") or provided
392
- # directly. Task descriptors are resolved using task.
393
- #
394
- # run returns the specified task on completion.
395
- def run(td=nil, *inputs)
653
+ # Sets state = State::READY unless the app has a run_thread
654
+ # (ie the app is running). Returns self.
655
+ def ready
656
+ synchronize do
657
+ self.state = State::READY if self.run_thread == nil
658
+ self
659
+ end
660
+ end
661
+
662
+ # Runs the methods in the queue in which they were enqued. Run exists when there
663
+ # are no more enqued methods. Run returns self. An app can only run on one thread
664
+ # at a time. If run is called when self is already running, run returns immediately.
665
+ #
666
+ # === The Run Cycle
667
+ # During run, each method is executed sequentially on the current thread unless
668
+ # m.multithread == true. In this case run switches into a multithreaded mode and
669
+ # launches up to n execution threads (where n is options.max_threads or
670
+ # DEFAULT_MAX_THREADS) each of which can run a multithreaded method.
671
+ #
672
+ # These threads will run methods until a non-multithreaded method reaches the top
673
+ # of the queue. At that point, run waits for the multithreaded methods to complete,
674
+ # and then switches back into the sequential mode. Run never executes multithreaded
675
+ # and non-multithreaded methods at the same time.
676
+ #
677
+ # Run checks the state of self before executing a method. If the state is changed
678
+ # to State::STOP, then no more methods will be executed (but currently running methods
679
+ # will continute to completion). If the state is changed to State::TERMINATE then
680
+ # no more methods will be executed and currently running methods will be discontinued
681
+ # as described below.
682
+ #
683
+ # When a series of multithreaded methods are stopped or terminated mid-execution,
684
+ # several methods may be waiting for a free execution thread. These are requeued.
685
+ #
686
+ # === Error Handling and Termination
687
+ # When unhandled errors arise during run, run enters a termination (rescue)
688
+ # routine. During termination a TerminationError is raised in each executing
689
+ # method so that the method exits or begins executing its internal error handling
690
+ # code (perhaps performing rollbacks).
691
+ #
692
+ # The TerminationError is ONLY raised when the method calls Task::Base#check_terminate
693
+ # This method is available to all Task::Base objects, but obviously is NOT available
694
+ # to Executable methods generated by _method. These methods need to check the state
695
+ # of app themselves; otherwise they will continue on to completion even when app
696
+ # is in State::TERMINATE.
697
+ #
698
+ # # this task will loop until app.terminate
699
+ # Task.new {|task| while(true) task.check_terminate end }
700
+ #
701
+ # # this task will NEVER terminate
702
+ # Task.new {|task| while(true) end; task.check_terminate }
703
+ #
704
+ # Additional errors that arise during termination are collected and packaged
705
+ # with the orignal error into a RunError. By default all errors are logged
706
+ # and run exits. If debug? == true, then the RunError will be raised for further
707
+ # handling.
708
+ #
709
+ # Note: the method that caused the original unhandled error is no longer executing
710
+ # when termination begins and thus will not recieve a TerminationError.
711
+ def run
396
712
  synchronize do
397
- # TODO: log starting run
398
-
399
- # lookup and enqueue the task, if provided
400
- specified_task = (td == nil || td.kind_of?(Tap::Task::Base) ? td : task(td))
401
- queue.enq(specified_task, *inputs) if specified_task
713
+ return self unless self.ready.state == State::READY
402
714
 
403
- # generate threading variables
715
+ self.run_thread = Thread.current
404
716
  self.state = State::RUN
405
- max_threads = options.max_threads || DEFAULT_MAX_THREADS
406
- self.thread_queue = max_threads > 0 ? Queue.new : nil
407
-
408
- begin
409
- execution_loop do
410
- task, inputs = queue.deq
411
-
412
- # if no tasks were in the queue
413
- # then clear the threads and
414
- # check for tasks again
415
- unless task
416
- clear_threads
417
- task, inputs = queue.deq
418
- end
419
-
420
- # break -- no executable task was found
421
- break if task.nil?
422
-
423
- if thread_queue && task.multithread?
424
- # TODO: log enqueuing task to thread
425
-
426
- # generate threads as needed and allowed
427
- # to execute the threads in the thread queue
428
- start_thread if threads.size < max_threads
429
-
430
- # NOTE: the producer-consumer relationship of execution
431
- # threads and the thread_queue means that tasks will sit
432
- # waiting until an execution thread opens up. in the most
433
- # extreme case all executing tasks and all tasks in the
434
- # task_queue could be the same task, each with different
435
- # inputs. this deviates from the idea of batch processing,
436
- # but should be rare and not at all fatal given execute
437
- # synchronization.
438
- thread_queue.enq [task, inputs]
439
-
440
- else
441
- # TODO: log execute task
442
-
443
- # wait for threads to complete
444
- # before executing the main thread
445
- clear_threads
446
- task.execute(*inputs)
447
- end
448
- end
717
+ end
718
+
719
+ # generate threading variables
720
+ max_threads = options.max_threads || DEFAULT_MAX_THREADS
721
+ self.thread_queue = max_threads > 0 ? Queue.new : nil
722
+
723
+ # TODO: log starting run
724
+ begin
725
+ execution_loop do
726
+ break if block_given? && yield(self)
449
727
 
450
- # if the run loop exited due to a STOP state,
451
- # tasks may still be in the thread queue and/or
452
- # running. be sure these are cleared
453
- if state == State::STOP
454
- clear_thread_queue
728
+ # if no tasks were in the queue
729
+ # then clear the threads and
730
+ # check for tasks again
731
+ if queue.empty?
455
732
  clear_threads
733
+ # break -- no executable task was found
734
+ break if queue.empty?
456
735
  end
457
736
 
458
- rescue
459
- # when an error is generated, be sure to terminate
460
- # all threads so they can clean up after themselves.
461
- # clear the thread queue first so no more tasks are
462
- # executed. collect any errors that arise during
463
- # termination.
464
- clear_thread_queue
465
- errors = clear_threads(false)
466
-
467
- # handle the errors accordingly
468
- if options.debug
469
- raise Tap::Support::RunError.new($!, errors)
737
+ m, inputs = queue.deq
738
+
739
+ if thread_queue && m.multithread
740
+ # TODO: log enqueuing task to thread
741
+
742
+ # generate threads as needed and allowed
743
+ # to execute the threads in the thread queue
744
+ start_thread if threads.size < max_threads
745
+
746
+ # NOTE: the producer-consumer relationship of execution
747
+ # threads and the thread_queue means that tasks will sit
748
+ # waiting until an execution thread opens up. in the most
749
+ # extreme case all executing tasks and all tasks in the
750
+ # task_queue could be the same task, each with different
751
+ # inputs. this deviates from the idea of batch processing,
752
+ # but should be rare and not at all fatal given execute
753
+ # synchronization.
754
+ thread_queue.enq [m, inputs]
755
+
470
756
  else
471
- log($!.class, $!.message)
472
- errors.each_with_index do |err, index|
473
- next if err.nil?
474
- log("[#{index}] #{err.class}", err.message)
475
- end
757
+ # TODO: log execute task
758
+
759
+ # wait for threads to complete
760
+ # before executing the main thread
761
+ clear_threads
762
+ execute(m, inputs)
476
763
  end
477
764
  end
478
765
 
766
+ # if the run loop exited due to a STOP state,
767
+ # tasks may still be in the thread queue and/or
768
+ # running. be sure these are cleared
769
+ clear_thread_queue
770
+ clear_threads
771
+
772
+ rescue
773
+ # when an error is generated, be sure to terminate
774
+ # all threads so they can clean up after themselves.
775
+ # clear the thread queue first so no more tasks are
776
+ # executed. collect any errors that arise during
777
+ # termination.
778
+ clear_thread_queue
779
+ errors = [$!] + clear_threads(false)
780
+ errors.delete_if {|error| error.kind_of?(TerminateError) }
781
+
782
+ # handle the errors accordingly
783
+ case
784
+ when debug?
785
+ raise Tap::Support::RunError.new(errors)
786
+ else
787
+ errors.each_with_index do |err, index|
788
+ log("RunError [#{index}] #{err.class}", err.message)
789
+ end
790
+ end
791
+ ensure
792
+
479
793
  # reset run variables
480
794
  self.thread_queue = nil
481
- self.state = State::READY
482
-
483
- # TODO: log run complete
484
795
 
485
- specified_task
796
+ synchronize do
797
+ self.run_thread = nil
798
+ self.state = State::READY
799
+ end
486
800
  end
801
+
802
+ # TODO: log run complete
803
+ self
487
804
  end
488
805
 
489
806
  # Signals a running application to stop executing tasks in the
@@ -492,115 +809,153 @@ module Tap
492
809
  #
493
810
  # Does nothing unless state is State::RUN.
494
811
  def stop
495
- self.state = State::STOP if state == State::RUN
812
+ synchronize do
813
+ self.state = State::STOP if self.state == State::RUN
814
+ self
815
+ end
496
816
  end
497
817
 
498
818
  # Signals a running application to terminate executing tasks
499
819
  # by setting state = State::TERMINATE. When running tasks
500
- # reach a termination check, the task will raise a Termination
501
- # error, thus allowing executing tasks to invoke their specific
820
+ # reach a termination check, the task raises a TerminationError,
821
+ # thus allowing executing tasks to invoke their specific
502
822
  # error handling code, perhaps performing rollbacks.
503
823
  #
504
824
  # Termination checks can be manually specified in a task
505
- # using the check_terminate method. Termination checks
506
- # automatically occur before each task execution.
825
+ # using the check_terminate method (see Tap::Task::Base#check_terminate).
826
+ # Termination checks automatically occur before each task execution.
507
827
  #
508
- # Does nothing unless state is State::RUN or State::STOP.
509
- def terminate#(error=TerminateError.new)
510
- if state == State::RUN || state == State::STOP
511
- self.state = State::TERMINATE
828
+ # Does nothing if state == State::READY.
829
+ def terminate
830
+ synchronize do
831
+ self.state = State::TERMINATE unless self.state == State::READY
832
+ self
512
833
  end
513
834
  end
514
835
 
515
836
  # Returns an information string for the App.
516
837
  #
517
- # App.new().info # => 'state: 0 (READY) queue: 0 waiting: 0 (0) threads: 0'
838
+ # App.instance.info # => 'state: 0 (READY) queue: 0 thread_queue: 0 threads: 0 results: 0'
518
839
  #
519
840
  # Provided information:
520
841
  #
521
- # state:: the integer and string values of the application state
522
- # queue:: the number of tasks currently in the queue
523
- # waiting:: the number of executable tasks in the queue, in parenthesis is the
524
- # number of objects in the thread queue (tasks, and perhaps nils to
525
- # signal threads to clear)
842
+ # state:: the integer and string values of self.state
843
+ # queue:: the number of methods currently in the queue
844
+ # thread_queue:: number of objects in the thread queue, waiting
845
+ # to be run on an execution thread (methods, and
846
+ # perhaps nils to signal threads to clear)
526
847
  # threads:: the number of execution threads
527
- #
848
+ # results:: the total number of results in aggregator
528
849
  def info
529
- state_str = State.constants.inject(nil) {|str, s| State.const_get(s) == state ? s : str}
530
- "state: #{state} (#{state_str}) queue: #{queue.size} waiting: #{queue.num_executable} (#{thread_queue ? thread_queue.size : 0}) threads: #{threads.size}"
850
+ synchronize do
851
+ "state: #{state} (#{State.state_str(state)}) queue: #{queue.size} thread_queue: #{thread_queue ? thread_queue.size : 0} threads: #{threads.size} results: #{aggregator.size}"
852
+ end
531
853
  end
532
854
 
533
855
  #
534
856
  # workflow related
535
857
  #
536
858
 
537
- # Sets the condition block for the task. If the task is batched,
538
- # then the condition will be set for each task in the batch.
539
- def condition(task, &block) # :yields: task, audited_inputs
540
- task.batch.each {|t| t.condition(&block)}
541
- end
542
-
543
- # Sets the on_complete block for the task. If the task is batched,
544
- # then on_complete will be set for each task in the batch.
545
- def on_complete(task, &block) # :yields: results
546
- task.batch.each {|t| t.on_complete(&block)}
547
- end
548
-
549
- # Sets multithread=true for each input task. Batched tasks will
550
- # have each task in the batch multithreaded.
551
- def multithread(*tasks)
552
- tasks.each do |task|
553
- task.batch.each {|t| t.multithread = true}
859
+ # Enques the task with the inputs. If the task is batched, then each
860
+ # task in task.batch will be enqued with the inputs. Returns task.
861
+ #
862
+ # An Executable may provided instead of a task.
863
+ def enq(task, *inputs)
864
+ case task
865
+ when Tap::Task::Base
866
+ raise "not assigned to enqueing app: #{task}" unless task.app == self
867
+ task.enq(*inputs)
868
+ when Support::Executable
869
+ queue.enq(task, inputs)
870
+ else
871
+ raise "Not a Task or Executable: #{task}"
554
872
  end
873
+ task
874
+ end
875
+
876
+ # Method enque. Enques the specified method from object with the inputs.
877
+ # Returns the enqued method.
878
+ def mq(object, method_name, *inputs)
879
+ m = object._method(method_name)
880
+ enq(m, *inputs)
555
881
  end
556
882
 
557
883
  # Sets a sequence workflow pattern for the tasks such that the
558
884
  # completion of a task enqueues the next task with it's results.
559
- # The condition block between these tasks will be set to the
560
- # input block, if provided. Batched tasks will have the pattern
561
- # set for each task in the batch.
562
- def sequence(*tasks, &block) # :yields: task, audited_inputs
885
+ # Batched tasks will have the pattern set for each task in the
886
+ # batch. The current audited results are yielded to the block,
887
+ # if given, before the next task is enqued.
888
+ #
889
+ # Executables may provided as well as tasks.
890
+ def sequence(*tasks) # :yields: _result
563
891
  current_task = tasks.shift
564
-
565
892
  tasks.each do |next_task|
566
893
  # simply pass results from one task to the next.
567
- on_complete(current_task) {|results| queue.enq(next_task, *results) }
568
- condition(next_task, &block)
894
+ current_task.on_complete do |_result|
895
+ yield(_result) if block_given?
896
+ enq(next_task, _result)
897
+ end
569
898
  current_task = next_task
570
899
  end
571
900
  end
572
901
 
573
902
  # Sets a fork workflow pattern for the tasks such that each of the
574
903
  # targets will be enqueued with the results of the source when the
575
- # source completes. The condition block for the targets will
576
- # be set to the input block, if provided. Batched tasks will have
577
- # the pattern set for each task in the batch.
578
- def fork(source, *targets, &block) # :yields: task, audited_inputs
579
- on_complete(source) do |results|
580
- targets.each do |target|
581
- # forking requires new audit trails because the same
582
- # inputs are getting sent to multiple tasks
583
- forked_results = results.collect {|result| result._fork }
584
- queue.enq(target, *forked_results)
904
+ # source completes. Batched tasks will have the pattern set for each
905
+ # task in the batch. The source audited results are yielded to the
906
+ # block, if given, before the targets are enqued.
907
+ #
908
+ # Executables may provided as well as tasks.
909
+ def fork(source, *targets) # :yields: _result
910
+ source.on_complete do |_result|
911
+ targets.each do |target|
912
+ yield(_result) if block_given?
913
+ enq(target, _result)
585
914
  end
586
915
  end
587
- targets.each do |target|
588
- condition(target, &block)
589
- end if block_given?
590
916
  end
591
917
 
592
918
  # Sets a merge workflow pattern for the tasks such that the results
593
919
  # of each source will be enqueued to the target when the source
594
- # completes. The condition block for the target will be set to the
595
- # input block, if provided. Batched tasks will have the pattern set
596
- # for each task in the batch.
597
- def merge(target, *sources, &block) # :yields: task, audited_inputs
920
+ # completes. Batched tasks will have the pattern set for each
921
+ # task in the batch. The source audited results are yielded to
922
+ # the block, if given, before the target is enqued.
923
+ #
924
+ # Executables may provided as well as tasks.
925
+ def merge(target, *sources) # :yields: _result
598
926
  sources.each do |source|
599
927
  # merging can use the existing audit trails... each distinct
600
928
  # input is getting sent to one place (the target)
601
- on_complete(source) {|results| queue.enq(target, *results) }
929
+ source.on_complete do |_result|
930
+ yield(_result) if block_given?
931
+ enq(target, _result)
932
+ end
602
933
  end
603
- condition(target, &block) if block_given?
934
+ end
935
+
936
+ # Returns all aggregated, audited results for the specified tasks.
937
+ # Results are joined into a single array. Arrays of tasks are
938
+ # allowed as inputs. See results.
939
+ def _results(*tasks)
940
+ aggregator.retrieve_all(*tasks.flatten)
941
+ end
942
+
943
+ # Returns all aggregated results for the specified tasks. Results are
944
+ # joined into a single array. Arrays of tasks are allowed as inputs.
945
+ #
946
+ # t1 = Task.new {|task, input| input += 1 }
947
+ # t2 = Task.new {|task, input| input += 10 }
948
+ # t3 = t2.initialize_batch_obj
949
+ #
950
+ # t1.enq(0)
951
+ # t2.enq(1)
952
+ #
953
+ # app.run
954
+ # app.results(t1, t2.batch) # => [1, 11, 11]
955
+ # app.results(t2, t1) # => [11, 1]
956
+ #
957
+ def results(*tasks)
958
+ _results(tasks).collect {|_result| _result._current}
604
959
  end
605
960
 
606
961
  protected
@@ -612,30 +967,34 @@ module Tap
612
967
  false
613
968
  end
614
969
 
970
+ # Sets the state of the application
615
971
  attr_writer :state
616
- attr_accessor :thread_queue, :threads
617
972
 
618
- private
973
+ # The thread on which run is executing tasks.
974
+ attr_accessor :run_thread
619
975
 
976
+ # An array containing the execution threads in use by run.
977
+ attr_accessor :threads
978
+
979
+ # A Queue containing multithread tasks waiting to be run
980
+ # on the execution threads. Nil if options.max_threads= 0
981
+ attr_accessor :thread_queue
982
+
983
+ private
984
+
620
985
  def execution_loop
621
986
  while true
622
987
  case state
623
988
  when State::STOP
624
989
  break
625
990
  when State::TERMINATE
626
- # if an execution thread (main or multi) handles the
627
- # termination error, then the thread may end up here --
628
- # terminated but still running. Raise another
629
- # termination error to enter the termination
630
- # (rescue) code.
631
-
632
- # BUG - if things just work out that way, terminate can
633
- # be set from a thread that had an error, then control
634
- # switches such that this is raised on the main thread.
635
- # in this case it looks like THIS is the original error.
991
+ # if an execution thread handles the termination error,
992
+ # then the thread may end up here -- terminated but still
993
+ # running. Raise another termination error to enter the
994
+ # termination (rescue) code.
636
995
  raise TerminateError.new
637
996
  end
638
-
997
+
639
998
  yield
640
999
  end
641
1000
  end
@@ -657,7 +1016,7 @@ module Tap
657
1016
  # task being promoted along with it's inputs
658
1017
  dequeued.reverse_each do |task, inputs|
659
1018
  # TODO: log about not executing
660
- queue.priority_enq(task, *inputs) unless task.nil?
1019
+ queue.unshift(task, inputs) unless task.nil?
661
1020
  end
662
1021
  end
663
1022
 
@@ -707,30 +1066,29 @@ module Tap
707
1066
 
708
1067
  begin
709
1068
  execution_loop do
710
- task, inputs = thread_queue.deq
711
- break if task.nil?
1069
+ m, inputs = thread_queue.deq
1070
+ break if m.nil?
712
1071
 
713
1072
  # TODO: log execute task on thread #
714
- task.execute(*inputs)
1073
+ execute(m, inputs)
715
1074
  end
716
1075
  rescue
717
- # unless you're already terminating,
718
1076
  # an unhandled error should immediately
719
1077
  # terminate all threads
720
- terminate unless state == State::TERMINATE
721
- Thread.current["error"] = $! unless $!.kind_of?(TerminateError)
1078
+ terminate
1079
+ Thread.current["error"] = $!
722
1080
  end
723
1081
  end
724
1082
  end
725
1083
  end
726
-
1084
+
727
1085
  # LookupErrors are raised for errors during dependency lookup
728
- class LookupError < RuntimeError # :nodoc:
1086
+ class LookupError < RuntimeError
729
1087
  end
730
1088
 
731
1089
  # TerminateErrors are raised to kill executing tasks when terminate
732
- # is called on an running App.
733
- class TerminateError < RuntimeError
1090
+ # is called on an running App. They are handled by the run rescue code.
1091
+ class TerminateError < RuntimeError
734
1092
  end
735
1093
  end
736
1094
  end