wukong 3.0.0.pre3 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. data/Gemfile +1 -0
  2. data/README.md +689 -50
  3. data/bin/wu-local +1 -74
  4. data/diagrams/wu_local.dot +39 -0
  5. data/diagrams/wu_local.dot.png +0 -0
  6. data/examples/loadable.rb +2 -0
  7. data/examples/string_reverser.rb +7 -0
  8. data/lib/hanuman/stage.rb +2 -2
  9. data/lib/wukong.rb +21 -10
  10. data/lib/wukong/dataflow.rb +2 -5
  11. data/lib/wukong/doc_helpers.rb +14 -0
  12. data/lib/wukong/doc_helpers/dataflow_handler.rb +29 -0
  13. data/lib/wukong/doc_helpers/field_handler.rb +91 -0
  14. data/lib/wukong/doc_helpers/processor_handler.rb +29 -0
  15. data/lib/wukong/driver.rb +11 -1
  16. data/lib/wukong/local.rb +40 -0
  17. data/lib/wukong/local/event_machine_driver.rb +27 -0
  18. data/lib/wukong/local/runner.rb +98 -0
  19. data/lib/wukong/local/stdio_driver.rb +44 -0
  20. data/lib/wukong/local/tcp_driver.rb +47 -0
  21. data/lib/wukong/logger.rb +16 -7
  22. data/lib/wukong/plugin.rb +48 -0
  23. data/lib/wukong/processor.rb +57 -15
  24. data/lib/wukong/rake_helper.rb +6 -0
  25. data/lib/wukong/runner.rb +151 -128
  26. data/lib/wukong/runner/boot_sequence.rb +123 -0
  27. data/lib/wukong/runner/code_loader.rb +52 -0
  28. data/lib/wukong/runner/deploy_pack_loader.rb +75 -0
  29. data/lib/wukong/runner/help_message.rb +42 -0
  30. data/lib/wukong/spec_helpers.rb +4 -12
  31. data/lib/wukong/spec_helpers/integration_tests.rb +150 -0
  32. data/lib/wukong/spec_helpers/{integration_driver_matchers.rb → integration_tests/integration_test_matchers.rb} +28 -62
  33. data/lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb +97 -0
  34. data/lib/wukong/spec_helpers/shared_examples.rb +19 -10
  35. data/lib/wukong/spec_helpers/unit_tests.rb +134 -0
  36. data/lib/wukong/spec_helpers/{processor_methods.rb → unit_tests/unit_test_driver.rb} +42 -8
  37. data/lib/wukong/spec_helpers/{spec_driver_matchers.rb → unit_tests/unit_test_matchers.rb} +6 -32
  38. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +54 -0
  39. data/lib/wukong/version.rb +1 -1
  40. data/lib/wukong/widget/filters.rb +134 -8
  41. data/lib/wukong/widget/processors.rb +64 -5
  42. data/lib/wukong/widget/reducers/bin.rb +68 -18
  43. data/lib/wukong/widget/reducers/count.rb +12 -0
  44. data/lib/wukong/widget/reducers/group.rb +48 -5
  45. data/lib/wukong/widget/reducers/group_concat.rb +30 -2
  46. data/lib/wukong/widget/reducers/moments.rb +4 -4
  47. data/lib/wukong/widget/reducers/sort.rb +53 -3
  48. data/lib/wukong/widget/serializers.rb +37 -12
  49. data/lib/wukong/widget/utils.rb +1 -1
  50. data/spec/spec_helper.rb +20 -2
  51. data/spec/wukong/driver_spec.rb +2 -0
  52. data/spec/wukong/local/runner_spec.rb +40 -0
  53. data/spec/wukong/local_spec.rb +6 -0
  54. data/spec/wukong/logger_spec.rb +49 -0
  55. data/spec/wukong/processor_spec.rb +22 -0
  56. data/spec/wukong/runner_spec.rb +128 -8
  57. data/spec/wukong/widget/filters_spec.rb +28 -10
  58. data/spec/wukong/widget/processors_spec.rb +5 -5
  59. data/spec/wukong/widget/reducers/bin_spec.rb +14 -14
  60. data/spec/wukong/widget/reducers/count_spec.rb +1 -1
  61. data/spec/wukong/widget/reducers/group_spec.rb +7 -6
  62. data/spec/wukong/widget/reducers/moments_spec.rb +2 -2
  63. data/spec/wukong/widget/reducers/sort_spec.rb +1 -1
  64. data/spec/wukong/widget/serializers_spec.rb +84 -88
  65. data/spec/wukong/wu-local_spec.rb +109 -0
  66. metadata +43 -20
  67. data/bin/wu-server +0 -70
  68. data/lib/wukong/boot.rb +0 -96
  69. data/lib/wukong/configuration.rb +0 -8
  70. data/lib/wukong/emitter.rb +0 -22
  71. data/lib/wukong/server.rb +0 -119
  72. data/lib/wukong/spec_helpers/integration_driver.rb +0 -157
  73. data/lib/wukong/spec_helpers/processor_helpers.rb +0 -89
  74. data/lib/wukong/spec_helpers/spec_driver.rb +0 -28
  75. data/spec/wukong/local_runner_spec.rb +0 -31
  76. data/spec/wukong/wu_local_spec.rb +0 -125
@@ -0,0 +1,75 @@
1
+ module Wukong
2
+ class Runner
3
+
4
+ # Lets Wukong bootstrap by requiring an enclosing deploy pack's
5
+ # environment file if available.
6
+ #
7
+ # We use a simple heuristic (presence of 'Gemfile' and
8
+ # 'config/environment.rb' in a non-root parent directory) to
9
+ # determine whether or not we are in a deploy pack.
10
+ module DeployPackLoader
11
+
12
+ # Load the actual deploy pack environment. Will not swallow any
13
+ # load errors.
14
+ def load_deploy_pack
15
+ load_ruby_file(environment_file) if in_deploy_pack?
16
+ end
17
+
18
+ # Is execution likely happening within a deploy pack?
19
+ #
20
+ # See Wukong::Deploy for more information on deploy packs.
21
+ #
22
+ # @return [true, false]
23
+ def in_deploy_pack?
24
+ return @in_deploy_pack unless @in_deploy_pack.nil?
25
+ @in_deploy_pack = (deploy_pack_dir != '/')
26
+ end
27
+
28
+ # Have we already loaded the environment of a deploy pack?
29
+ #
30
+ # See Wukong::Deploy for more information on deploy packs.
31
+ #
32
+ # @return [true, false]
33
+ def loaded_deploy_pack?
34
+ in_deploy_pack? && defined?(::Wukong::Deploy)
35
+ end
36
+
37
+ # The default environment file that will be require'd when
38
+ # booting.
39
+ #
40
+ # @return [String]
41
+ def environment_file
42
+ File.join(deploy_pack_dir, 'config', 'environment.rb')
43
+ end
44
+
45
+ # Return the directory of the enclosing deploy pack. Will return
46
+ # the root ('/') if no deeper directory is identified as a deploy
47
+ # pack.
48
+ #
49
+ # @return [String]
50
+ def deploy_pack_dir
51
+ return File.dirname(ENV["BUNDLE_GEMFILE"]) if ENV["BUNDLE_GEMFILE"] && is_deploy_pack_dir?(File.dirname(ENV["BUNDLE_GEMFILE"]))
52
+ return @deploy_pack_dir if @deploy_pack_dir
53
+ wd = Dir.pwd
54
+ parent = File.dirname(wd)
55
+ until wd == parent
56
+ return wd if is_deploy_pack_dir?(wd)
57
+ wd = parent
58
+ parent = File.dirname(wd)
59
+ end
60
+ @deploy_pack_dir = wd
61
+ end
62
+
63
+ private
64
+
65
+ # Could `dir` be a deploy pack dir?
66
+ #
67
+ # @param [String] dir
68
+ # @return [true, false]
69
+ def is_deploy_pack_dir? dir
70
+ dir && !dir.empty? && File.directory?(dir) && File.exist?(File.join(dir, 'Gemfile')) && File.exist?(File.join(dir, 'config', 'environment.rb'))
71
+ end
72
+ end
73
+
74
+ end
75
+ end
@@ -0,0 +1,42 @@
1
+ module Wukong
2
+ class Runner
3
+
4
+ # Defines methods for handling help messages.
5
+ #
6
+ # Runners which want to modify how help messages are generated
7
+ # should override the `contextualize_help_message!` instance method.
8
+ module HelpMessage
9
+
10
+ # Was the `--help` option specified on the command line?
11
+ #
12
+ # The boot sequence for a Runner strips out the `--help` option to
13
+ # allow individual Runner classes to customize their help
14
+ # messages.
15
+ #
16
+ # @return [true, false]
17
+ def help_given?
18
+ !!@help_given
19
+ end
20
+
21
+ # Strip the `--help` message from the original ARGV, storing
22
+ # whether or not it was given for later.
23
+ def strip_help_param!
24
+ @help_given = ARGV.delete('--help')
25
+ end
26
+
27
+ # Print a help message.
28
+ def dump_help
29
+ settings.dump_help
30
+ end
31
+
32
+ # Print a help message and exit.
33
+ #
34
+ # @raise [SystemExit]
35
+ def dump_help_and_exit!
36
+ dump_help
37
+ exit(1)
38
+ end
39
+
40
+ end
41
+ end
42
+ end
@@ -1,10 +1,6 @@
1
1
  require 'wukong'
2
- require 'wukong/boot'
3
- require_relative('spec_helpers/processor_helpers')
4
- require_relative('spec_helpers/processor_methods')
5
- require_relative('spec_helpers/spec_driver_matchers')
6
- require_relative('spec_helpers/integration_driver')
7
- require_relative('spec_helpers/integration_driver_matchers')
2
+ require_relative('spec_helpers/unit_tests')
3
+ require_relative('spec_helpers/integration_tests')
8
4
  require_relative('spec_helpers/shared_examples')
9
5
 
10
6
  module Wukong
@@ -71,12 +67,8 @@ module Wukong
71
67
  # processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
72
68
  # end
73
69
  module SpecHelpers
74
- include ProcessorHelpers
75
- include SpecMatchers
76
- include IntegrationRunner
77
- include IntegrationMatchers
70
+ include UnitTests
71
+ include IntegrationTests
78
72
  end
79
-
80
- Processor.class_eval { include SpecHelpers::ProcessorSpecMethods }
81
73
  end
82
74
 
@@ -0,0 +1,150 @@
1
+ require_relative('integration_tests/integration_test_runner')
2
+ require_relative('integration_tests/integration_test_matchers')
3
+
4
+ module Wukong
5
+ module SpecHelpers
6
+
7
+ # This module defines methods that are helpful to use in
8
+ # integration tests which require reading files from the local
9
+ # repository.
10
+ #
11
+ # Integration tests will spawn new system processes with their own
12
+ # environments. This module provides methods and hooks for
13
+ # customizing that environment.
14
+ module IntegrationTests
15
+
16
+ # The directory to add to the `RUBYLIB` environment variable for
17
+ # the spawned processes.
18
+ #
19
+ # If `args` are given, return a path within this directory.
20
+ #
21
+ # @param [Array<String>] args
22
+ # @return [String]
23
+ def lib_dir *args
24
+ root.join('lib', *args).to_s
25
+ end
26
+
27
+ # The directory to add to the `PATH` environment variable for
28
+ # the spawned processes.
29
+ #
30
+ # If `args` are given, return a path within this directory.
31
+ #
32
+ # @param [Array<String>] args
33
+ # @return [String]
34
+ def bin_dir *args
35
+ root.join('bin', *args).to_s
36
+ end
37
+
38
+ # The directory to use for examples for the spawned process.
39
+ #
40
+ # If `args` are given, return a path within this directory.
41
+ #
42
+ # @param [Array<String>] args
43
+ # @return [String]
44
+ def examples_dir *args
45
+ root.join('examples', *args).to_s
46
+ end
47
+
48
+ # A Hash of environment variables to use for the spawned
49
+ # process.
50
+ #
51
+ # By default, will put the IntegrationHelper#lib_dir on the
52
+ # `RUBYLIB` and the IntegrationHelper#bin_dir on the `PATH`.
53
+ #
54
+ # @return [Hash]
55
+ def integration_env
56
+ {
57
+ "PATH" => [bin_dir.to_s, ENV["PATH"]].compact.join(':'),
58
+ "RUBYLIB" => [lib_dir.to_s, ENV["RUBYLIB"]].compact.join(':')
59
+ }
60
+ end
61
+
62
+ # The directory to spawn new processes in.
63
+ #
64
+ # @return [String]
65
+ def integration_cwd
66
+ root.to_s
67
+ end
68
+
69
+ # Checks that each `expectation` appears in the STDOUT of the
70
+ # command. Order is irrelevant and each `expectation` can be
71
+ # either a String to check for inclusion or a Regexp to match
72
+ # with.
73
+ #
74
+ # @param [Array<String,Regexp>] expectations
75
+ def have_stdout *expectations
76
+ StdoutMatcher.new(*expectations)
77
+ end
78
+
79
+ # Checks that each `expectation` appears in the STDERR of the
80
+ # command. Order is irrelevant and each `expectation` can be
81
+ # either a String to check for inclusion or a Regexp to match
82
+ # with.
83
+ #
84
+ # @param [Array<String,Regexp>] expectations
85
+ def have_stderr *expectations
86
+ StderrMatcher.new(*expectations)
87
+ end
88
+
89
+ # Checks that the command exits with the given `code`.
90
+ #
91
+ # @param [Integer] code
92
+ def exit_with code
93
+ ExitCodeMatcher.new(code)
94
+ end
95
+
96
+ # Spawn a command and capture its STDOUT, STDERR, and exit code.
97
+ #
98
+ # The `args` will be joined together into a command line.
99
+ #
100
+ # It is expected that you will use the matchers defined in
101
+ # IntegrationMatchers in your integration tests:
102
+ #
103
+ # @example Check output of 'ls' includes a string 'foo.txt'
104
+ # it "lists files" do
105
+ # command('ls').should have_output('foo.txt')
106
+ # end
107
+ #
108
+ # @example More complicated
109
+ # context "long format" do
110
+ # it "lists files with timestamps" do
111
+ # command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
112
+ # end
113
+ # end
114
+ #
115
+ # @param [Array<String>] args
116
+ #
117
+ # @overload command(*args, options={})
118
+ # If the last element of `args` is a Hash it will be used for
119
+ # options.
120
+ #
121
+ # The :env option specifies the command line environment to
122
+ # use for the command. By default this will be the value of
123
+ # the Ruby process's own `ENV` variable. If running in a
124
+ # context in which the `integration_env` method is defined,
125
+ # its return value will be merged on top of `ENV`. An
126
+ # explicitly provided :env option will again be merged on top.
127
+ #
128
+ # The :cwd option specifies the working directory to start in.
129
+ # It defaults to the value of <tt>Dir.pwd</tt>
130
+ #
131
+ # @param [Array<String>] args
132
+ # @param [Hash] options
133
+ # @option options [Hash] env the shell environment to spawn the command with
134
+ # @option options [Hash] cwd the directory to execute the command in
135
+ def command *args
136
+ a = args.flatten.compact
137
+ options = (a.last.is_a?(Hash) ? a.pop : {})
138
+
139
+ env = ENV.to_hash.dup
140
+ env.merge!(integration_env) if respond_to?(:integration_env)
141
+ env.merge!(options[:env] || {})
142
+
143
+ cwd = options[:cwd]
144
+ cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
145
+
146
+ IntegrationTestRunner.new(a, cwd: cwd, env: env)
147
+ end
148
+ end
149
+ end
150
+ end
@@ -1,48 +1,15 @@
1
1
  module Wukong
2
2
  module SpecHelpers
3
-
4
- # Provides matchers for STDOUT, STDERR, and exit code when writing
5
- # integration tests for Wukong's command-line APIs.
6
- module IntegrationMatchers
7
-
8
- # Checks that each `expectation` appears in the STDOUT of the
9
- # command. Order is irrelevant and each `expectation` can be
10
- # either a String to check for inclusion or a Regexp to match
11
- # with.
12
- #
13
- # @param [Array<String,Regexp>] expectations
14
- def have_stdout *expectations
15
- StdoutMatcher.new(*expectations)
16
- end
17
-
18
- # Checks that each `expectation` appears in the STDERR of the
19
- # command. Order is irrelevant and each `expectation` can be
20
- # either a String to check for inclusion or a Regexp to match
21
- # with.
22
- #
23
- # @param [Array<String,Regexp>] expectations
24
- def have_stderr *expectations
25
- StderrMatcher.new(*expectations)
26
- end
27
-
28
- # Checks that the command exits with the given `code`.
29
- #
30
- # @param [Integer] code
31
- def exit_with code
32
- ExitCodeMatcher.new(code)
33
- end
34
-
35
- end
36
-
3
+
37
4
  # A class for running commands and capturing their STDOUT, STDERR,
38
5
  # and exit code. This class is designed to work with the matchers
39
- # defined in IntegrationMatchers.
40
- class IntegrationMatcher
6
+ # defined in IntegrationTestMatchers.
7
+ class IntegrationTestMatcher
41
8
 
42
- # The driver used to run the actual commands.
43
- attr_accessor :driver
9
+ # The runner used to run the actual commands.
10
+ attr_accessor :runner
44
11
 
45
- # An array of expectations about the output of the driver.
12
+ # An array of expectations about the output of the runner.
46
13
  attr_accessor :expectations
47
14
 
48
15
  # The expectation which caused failure.
@@ -54,11 +21,11 @@ module Wukong
54
21
  # If an expectation failes to match, the `failed_expectation`
55
22
  # attribute will be set accordingly.
56
23
  #
57
- # @param [IntegrationDriver] driver
24
+ # @param [IntegrationTestRunner] runner
58
25
  # @return [true, false]
59
- def matches?(driver)
60
- self.driver = driver
61
- driver.run!
26
+ def matches?(runner)
27
+ self.runner = runner
28
+ runner.run!
62
29
  expectations.each do |expectation|
63
30
  unless output.send(match_function(expectation), expectation)
64
31
  self.failed_expectation = expectation
@@ -80,12 +47,12 @@ module Wukong
80
47
 
81
48
  # :nodoc:
82
49
  def failure_message
83
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
50
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
84
51
  end
85
52
 
86
53
  # :nodoc:
87
54
  def negative_failure_message
88
- "Expected #{output_description} of #{driver.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
55
+ "Expected #{output_description} of #{runner.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
89
56
  end
90
57
 
91
58
  # :nodoc:
@@ -95,18 +62,18 @@ module Wukong
95
62
 
96
63
  # :nodoc:
97
64
  def formatted_error_output
98
- output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{driver.stdout}" : "\n\nSTDERR was\n\n#{driver.stderr}"
65
+ output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{runner.stdout}" : "\n\nSTDERR was\n\n#{runner.stderr}"
99
66
  end
100
67
 
101
68
  # :nodoc:
102
69
  def formatted_command
103
- " $ #{driver.cmd}"
70
+ " $ #{runner.cmd}"
104
71
  end
105
72
 
106
73
  # :nodoc:
107
74
  def formatted_env
108
75
  [' {'].tap do |lines|
109
- driver.env.each_pair do |key, value|
76
+ runner.env.each_pair do |key, value|
110
77
  if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
111
78
  lines << " #{key} => #{value},"
112
79
  end
@@ -128,11 +95,11 @@ module Wukong
128
95
  end
129
96
 
130
97
  # A matcher for the STDOUT of a command.
131
- class StdoutMatcher < IntegrationMatcher
98
+ class StdoutMatcher < IntegrationTestMatcher
132
99
 
133
100
  # Picks the STDOUT of the command.
134
101
  def output
135
- driver.stdout
102
+ runner.stdout
136
103
  end
137
104
 
138
105
  # :nodoc:
@@ -147,11 +114,11 @@ module Wukong
147
114
  end
148
115
 
149
116
  # A matcher for the STDOUT of a command.
150
- class StderrMatcher < IntegrationMatcher
117
+ class StderrMatcher < IntegrationTestMatcher
151
118
 
152
119
  # Picks the STDOUT of the command.
153
120
  def output
154
- driver.stderr
121
+ runner.stderr
155
122
  end
156
123
 
157
124
  # :nodoc:
@@ -165,7 +132,7 @@ module Wukong
165
132
  end
166
133
 
167
134
  # A matcher for the exit code of a command.
168
- class ExitCodeMatcher < IntegrationMatcher
135
+ class ExitCodeMatcher < IntegrationTestMatcher
169
136
 
170
137
  # Initialize this matcher with the given `code`.
171
138
  #
@@ -184,27 +151,27 @@ module Wukong
184
151
  # Return whether or not the given command's exit code matches
185
152
  # the expectation.
186
153
  #
187
- # @param [IntegrationDriver] driver
154
+ # @param [IntegrationTestRunner] runner
188
155
  # @return [true, false]
189
- def matches?(driver)
190
- self.driver = driver
191
- driver.run!
156
+ def matches?(runner)
157
+ self.runner = runner
158
+ runner.run!
192
159
  if non_zero_exit_code?
193
- @failed = true if driver.exit_code == 0
160
+ @failed = true if runner.exit_code == 0
194
161
  else
195
- @failed = true if driver.exit_code != expected_exit_code
162
+ @failed = true if runner.exit_code != expected_exit_code
196
163
  end
197
164
  @failed ? false : true
198
165
  end
199
166
 
200
167
  # :nodoc:
201
168
  def failure_message
202
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{driver.exit_code} instead.#{formatted_error_output}"
169
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{runner.exit_code} instead.#{formatted_error_output}"
203
170
  end
204
171
 
205
172
  # :nodoc:
206
173
  def negative_failure_message
207
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
174
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
208
175
  end
209
176
 
210
177
  # :nodoc:
@@ -235,7 +202,6 @@ module Wukong
235
202
  def output_description
236
203
  "STDOUT"
237
204
  end
238
-
239
205
  end
240
206
  end
241
207
  end