wukong 3.0.0.pre3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. data/Gemfile +1 -0
  2. data/README.md +689 -50
  3. data/bin/wu-local +1 -74
  4. data/diagrams/wu_local.dot +39 -0
  5. data/diagrams/wu_local.dot.png +0 -0
  6. data/examples/loadable.rb +2 -0
  7. data/examples/string_reverser.rb +7 -0
  8. data/lib/hanuman/stage.rb +2 -2
  9. data/lib/wukong.rb +21 -10
  10. data/lib/wukong/dataflow.rb +2 -5
  11. data/lib/wukong/doc_helpers.rb +14 -0
  12. data/lib/wukong/doc_helpers/dataflow_handler.rb +29 -0
  13. data/lib/wukong/doc_helpers/field_handler.rb +91 -0
  14. data/lib/wukong/doc_helpers/processor_handler.rb +29 -0
  15. data/lib/wukong/driver.rb +11 -1
  16. data/lib/wukong/local.rb +40 -0
  17. data/lib/wukong/local/event_machine_driver.rb +27 -0
  18. data/lib/wukong/local/runner.rb +98 -0
  19. data/lib/wukong/local/stdio_driver.rb +44 -0
  20. data/lib/wukong/local/tcp_driver.rb +47 -0
  21. data/lib/wukong/logger.rb +16 -7
  22. data/lib/wukong/plugin.rb +48 -0
  23. data/lib/wukong/processor.rb +57 -15
  24. data/lib/wukong/rake_helper.rb +6 -0
  25. data/lib/wukong/runner.rb +151 -128
  26. data/lib/wukong/runner/boot_sequence.rb +123 -0
  27. data/lib/wukong/runner/code_loader.rb +52 -0
  28. data/lib/wukong/runner/deploy_pack_loader.rb +75 -0
  29. data/lib/wukong/runner/help_message.rb +42 -0
  30. data/lib/wukong/spec_helpers.rb +4 -12
  31. data/lib/wukong/spec_helpers/integration_tests.rb +150 -0
  32. data/lib/wukong/spec_helpers/{integration_driver_matchers.rb → integration_tests/integration_test_matchers.rb} +28 -62
  33. data/lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb +97 -0
  34. data/lib/wukong/spec_helpers/shared_examples.rb +19 -10
  35. data/lib/wukong/spec_helpers/unit_tests.rb +134 -0
  36. data/lib/wukong/spec_helpers/{processor_methods.rb → unit_tests/unit_test_driver.rb} +42 -8
  37. data/lib/wukong/spec_helpers/{spec_driver_matchers.rb → unit_tests/unit_test_matchers.rb} +6 -32
  38. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +54 -0
  39. data/lib/wukong/version.rb +1 -1
  40. data/lib/wukong/widget/filters.rb +134 -8
  41. data/lib/wukong/widget/processors.rb +64 -5
  42. data/lib/wukong/widget/reducers/bin.rb +68 -18
  43. data/lib/wukong/widget/reducers/count.rb +12 -0
  44. data/lib/wukong/widget/reducers/group.rb +48 -5
  45. data/lib/wukong/widget/reducers/group_concat.rb +30 -2
  46. data/lib/wukong/widget/reducers/moments.rb +4 -4
  47. data/lib/wukong/widget/reducers/sort.rb +53 -3
  48. data/lib/wukong/widget/serializers.rb +37 -12
  49. data/lib/wukong/widget/utils.rb +1 -1
  50. data/spec/spec_helper.rb +20 -2
  51. data/spec/wukong/driver_spec.rb +2 -0
  52. data/spec/wukong/local/runner_spec.rb +40 -0
  53. data/spec/wukong/local_spec.rb +6 -0
  54. data/spec/wukong/logger_spec.rb +49 -0
  55. data/spec/wukong/processor_spec.rb +22 -0
  56. data/spec/wukong/runner_spec.rb +128 -8
  57. data/spec/wukong/widget/filters_spec.rb +28 -10
  58. data/spec/wukong/widget/processors_spec.rb +5 -5
  59. data/spec/wukong/widget/reducers/bin_spec.rb +14 -14
  60. data/spec/wukong/widget/reducers/count_spec.rb +1 -1
  61. data/spec/wukong/widget/reducers/group_spec.rb +7 -6
  62. data/spec/wukong/widget/reducers/moments_spec.rb +2 -2
  63. data/spec/wukong/widget/reducers/sort_spec.rb +1 -1
  64. data/spec/wukong/widget/serializers_spec.rb +84 -88
  65. data/spec/wukong/wu-local_spec.rb +109 -0
  66. metadata +43 -20
  67. data/bin/wu-server +0 -70
  68. data/lib/wukong/boot.rb +0 -96
  69. data/lib/wukong/configuration.rb +0 -8
  70. data/lib/wukong/emitter.rb +0 -22
  71. data/lib/wukong/server.rb +0 -119
  72. data/lib/wukong/spec_helpers/integration_driver.rb +0 -157
  73. data/lib/wukong/spec_helpers/processor_helpers.rb +0 -89
  74. data/lib/wukong/spec_helpers/spec_driver.rb +0 -28
  75. data/spec/wukong/local_runner_spec.rb +0 -31
  76. data/spec/wukong/wu_local_spec.rb +0 -125
@@ -0,0 +1,75 @@
1
+ module Wukong
2
+ class Runner
3
+
4
+ # Lets Wukong bootstrap by requiring an enclosing deploy pack's
5
+ # environment file if available.
6
+ #
7
+ # We use a simple heuristic (presence of 'Gemfile' and
8
+ # 'config/environment.rb' in a non-root parent directory) to
9
+ # determine whether or not we are in a deploy pack.
10
+ module DeployPackLoader
11
+
12
+ # Load the actual deploy pack environment. Will not swallow any
13
+ # load errors.
14
+ def load_deploy_pack
15
+ load_ruby_file(environment_file) if in_deploy_pack?
16
+ end
17
+
18
+ # Is execution likely happening within a deploy pack?
19
+ #
20
+ # See Wukong::Deploy for more information on deploy packs.
21
+ #
22
+ # @return [true, false]
23
+ def in_deploy_pack?
24
+ return @in_deploy_pack unless @in_deploy_pack.nil?
25
+ @in_deploy_pack = (deploy_pack_dir != '/')
26
+ end
27
+
28
+ # Have we already loaded the environment of a deploy pack?
29
+ #
30
+ # See Wukong::Deploy for more information on deploy packs.
31
+ #
32
+ # @return [true, false]
33
+ def loaded_deploy_pack?
34
+ in_deploy_pack? && defined?(::Wukong::Deploy)
35
+ end
36
+
37
+ # The default environment file that will be require'd when
38
+ # booting.
39
+ #
40
+ # @return [String]
41
+ def environment_file
42
+ File.join(deploy_pack_dir, 'config', 'environment.rb')
43
+ end
44
+
45
+ # Return the directory of the enclosing deploy pack. Will return
46
+ # the root ('/') if no deeper directory is identified as a deploy
47
+ # pack.
48
+ #
49
+ # @return [String]
50
+ def deploy_pack_dir
51
+ return File.dirname(ENV["BUNDLE_GEMFILE"]) if ENV["BUNDLE_GEMFILE"] && is_deploy_pack_dir?(File.dirname(ENV["BUNDLE_GEMFILE"]))
52
+ return @deploy_pack_dir if @deploy_pack_dir
53
+ wd = Dir.pwd
54
+ parent = File.dirname(wd)
55
+ until wd == parent
56
+ return wd if is_deploy_pack_dir?(wd)
57
+ wd = parent
58
+ parent = File.dirname(wd)
59
+ end
60
+ @deploy_pack_dir = wd
61
+ end
62
+
63
+ private
64
+
65
+ # Could `dir` be a deploy pack dir?
66
+ #
67
+ # @param [String] dir
68
+ # @return [true, false]
69
+ def is_deploy_pack_dir? dir
70
+ dir && !dir.empty? && File.directory?(dir) && File.exist?(File.join(dir, 'Gemfile')) && File.exist?(File.join(dir, 'config', 'environment.rb'))
71
+ end
72
+ end
73
+
74
+ end
75
+ end
@@ -0,0 +1,42 @@
1
+ module Wukong
2
+ class Runner
3
+
4
+ # Defines methods for handling help messages.
5
+ #
6
+ # Runners which want to modify how help messages are generated
7
+ # should override the `contextualize_help_message!` instance method.
8
+ module HelpMessage
9
+
10
+ # Was the `--help` option specified on the command line?
11
+ #
12
+ # The boot sequence for a Runner strips out the `--help` option to
13
+ # allow individual Runner classes to customize their help
14
+ # messages.
15
+ #
16
+ # @return [true, false]
17
+ def help_given?
18
+ !!@help_given
19
+ end
20
+
21
+ # Strip the `--help` message from the original ARGV, storing
22
+ # whether or not it was given for later.
23
+ def strip_help_param!
24
+ @help_given = ARGV.delete('--help')
25
+ end
26
+
27
+ # Print a help message.
28
+ def dump_help
29
+ settings.dump_help
30
+ end
31
+
32
+ # Print a help message and exit.
33
+ #
34
+ # @raise [SystemExit]
35
+ def dump_help_and_exit!
36
+ dump_help
37
+ exit(1)
38
+ end
39
+
40
+ end
41
+ end
42
+ end
@@ -1,10 +1,6 @@
1
1
  require 'wukong'
2
- require 'wukong/boot'
3
- require_relative('spec_helpers/processor_helpers')
4
- require_relative('spec_helpers/processor_methods')
5
- require_relative('spec_helpers/spec_driver_matchers')
6
- require_relative('spec_helpers/integration_driver')
7
- require_relative('spec_helpers/integration_driver_matchers')
2
+ require_relative('spec_helpers/unit_tests')
3
+ require_relative('spec_helpers/integration_tests')
8
4
  require_relative('spec_helpers/shared_examples')
9
5
 
10
6
  module Wukong
@@ -71,12 +67,8 @@ module Wukong
71
67
  # processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
72
68
  # end
73
69
  module SpecHelpers
74
- include ProcessorHelpers
75
- include SpecMatchers
76
- include IntegrationRunner
77
- include IntegrationMatchers
70
+ include UnitTests
71
+ include IntegrationTests
78
72
  end
79
-
80
- Processor.class_eval { include SpecHelpers::ProcessorSpecMethods }
81
73
  end
82
74
 
@@ -0,0 +1,150 @@
1
+ require_relative('integration_tests/integration_test_runner')
2
+ require_relative('integration_tests/integration_test_matchers')
3
+
4
+ module Wukong
5
+ module SpecHelpers
6
+
7
+ # This module defines methods that are helpful to use in
8
+ # integration tests which require reading files from the local
9
+ # repository.
10
+ #
11
+ # Integration tests will spawn new system processes with their own
12
+ # environments. This module provides methods and hooks for
13
+ # customizing that environment.
14
+ module IntegrationTests
15
+
16
+ # The directory to add to the `RUBYLIB` environment variable for
17
+ # the spawned processes.
18
+ #
19
+ # If `args` are given, return a path within this directory.
20
+ #
21
+ # @param [Array<String>] args
22
+ # @return [String]
23
+ def lib_dir *args
24
+ root.join('lib', *args).to_s
25
+ end
26
+
27
+ # The directory to add to the `PATH` environment variable for
28
+ # the spawned processes.
29
+ #
30
+ # If `args` are given, return a path within this directory.
31
+ #
32
+ # @param [Array<String>] args
33
+ # @return [String]
34
+ def bin_dir *args
35
+ root.join('bin', *args).to_s
36
+ end
37
+
38
+ # The directory to use for examples for the spawned process.
39
+ #
40
+ # If `args` are given, return a path within this directory.
41
+ #
42
+ # @param [Array<String>] args
43
+ # @return [String]
44
+ def examples_dir *args
45
+ root.join('examples', *args).to_s
46
+ end
47
+
48
+ # A Hash of environment variables to use for the spawned
49
+ # process.
50
+ #
51
+ # By default, will put the IntegrationHelper#lib_dir on the
52
+ # `RUBYLIB` and the IntegrationHelper#bin_dir on the `PATH`.
53
+ #
54
+ # @return [Hash]
55
+ def integration_env
56
+ {
57
+ "PATH" => [bin_dir.to_s, ENV["PATH"]].compact.join(':'),
58
+ "RUBYLIB" => [lib_dir.to_s, ENV["RUBYLIB"]].compact.join(':')
59
+ }
60
+ end
61
+
62
+ # The directory to spawn new processes in.
63
+ #
64
+ # @return [String]
65
+ def integration_cwd
66
+ root.to_s
67
+ end
68
+
69
+ # Checks that each `expectation` appears in the STDOUT of the
70
+ # command. Order is irrelevant and each `expectation` can be
71
+ # either a String to check for inclusion or a Regexp to match
72
+ # with.
73
+ #
74
+ # @param [Array<String,Regexp>] expectations
75
+ def have_stdout *expectations
76
+ StdoutMatcher.new(*expectations)
77
+ end
78
+
79
+ # Checks that each `expectation` appears in the STDERR of the
80
+ # command. Order is irrelevant and each `expectation` can be
81
+ # either a String to check for inclusion or a Regexp to match
82
+ # with.
83
+ #
84
+ # @param [Array<String,Regexp>] expectations
85
+ def have_stderr *expectations
86
+ StderrMatcher.new(*expectations)
87
+ end
88
+
89
+ # Checks that the command exits with the given `code`.
90
+ #
91
+ # @param [Integer] code
92
+ def exit_with code
93
+ ExitCodeMatcher.new(code)
94
+ end
95
+
96
+ # Spawn a command and capture its STDOUT, STDERR, and exit code.
97
+ #
98
+ # The `args` will be joined together into a command line.
99
+ #
100
+ # It is expected that you will use the matchers defined in
101
+ # IntegrationMatchers in your integration tests:
102
+ #
103
+ # @example Check output of 'ls' includes a string 'foo.txt'
104
+ # it "lists files" do
105
+ # command('ls').should have_output('foo.txt')
106
+ # end
107
+ #
108
+ # @example More complicated
109
+ # context "long format" do
110
+ # it "lists files with timestamps" do
111
+ # command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
112
+ # end
113
+ # end
114
+ #
115
+ # @param [Array<String>] args
116
+ #
117
+ # @overload command(*args, options={})
118
+ # If the last element of `args` is a Hash it will be used for
119
+ # options.
120
+ #
121
+ # The :env option specifies the command line environment to
122
+ # use for the command. By default this will be the value of
123
+ # the Ruby process's own `ENV` variable. If running in a
124
+ # context in which the `integration_env` method is defined,
125
+ # its return value will be merged on top of `ENV`. An
126
+ # explicitly provided :env option will again be merged on top.
127
+ #
128
+ # The :cwd option specifies the working directory to start in.
129
+ # It defaults to the value of <tt>Dir.pwd</tt>
130
+ #
131
+ # @param [Array<String>] args
132
+ # @param [Hash] options
133
+ # @option options [Hash] env the shell environment to spawn the command with
134
+ # @option options [Hash] cwd the directory to execute the command in
135
+ def command *args
136
+ a = args.flatten.compact
137
+ options = (a.last.is_a?(Hash) ? a.pop : {})
138
+
139
+ env = ENV.to_hash.dup
140
+ env.merge!(integration_env) if respond_to?(:integration_env)
141
+ env.merge!(options[:env] || {})
142
+
143
+ cwd = options[:cwd]
144
+ cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
145
+
146
+ IntegrationTestRunner.new(a, cwd: cwd, env: env)
147
+ end
148
+ end
149
+ end
150
+ end
@@ -1,48 +1,15 @@
1
1
  module Wukong
2
2
  module SpecHelpers
3
-
4
- # Provides matchers for STDOUT, STDERR, and exit code when writing
5
- # integration tests for Wukong's command-line APIs.
6
- module IntegrationMatchers
7
-
8
- # Checks that each `expectation` appears in the STDOUT of the
9
- # command. Order is irrelevant and each `expectation` can be
10
- # either a String to check for inclusion or a Regexp to match
11
- # with.
12
- #
13
- # @param [Array<String,Regexp>] expectations
14
- def have_stdout *expectations
15
- StdoutMatcher.new(*expectations)
16
- end
17
-
18
- # Checks that each `expectation` appears in the STDERR of the
19
- # command. Order is irrelevant and each `expectation` can be
20
- # either a String to check for inclusion or a Regexp to match
21
- # with.
22
- #
23
- # @param [Array<String,Regexp>] expectations
24
- def have_stderr *expectations
25
- StderrMatcher.new(*expectations)
26
- end
27
-
28
- # Checks that the command exits with the given `code`.
29
- #
30
- # @param [Integer] code
31
- def exit_with code
32
- ExitCodeMatcher.new(code)
33
- end
34
-
35
- end
36
-
3
+
37
4
  # A class for running commands and capturing their STDOUT, STDERR,
38
5
  # and exit code. This class is designed to work with the matchers
39
- # defined in IntegrationMatchers.
40
- class IntegrationMatcher
6
+ # defined in IntegrationTestMatchers.
7
+ class IntegrationTestMatcher
41
8
 
42
- # The driver used to run the actual commands.
43
- attr_accessor :driver
9
+ # The runner used to run the actual commands.
10
+ attr_accessor :runner
44
11
 
45
- # An array of expectations about the output of the driver.
12
+ # An array of expectations about the output of the runner.
46
13
  attr_accessor :expectations
47
14
 
48
15
  # The expectation which caused failure.
@@ -54,11 +21,11 @@ module Wukong
54
21
  # If an expectation failes to match, the `failed_expectation`
55
22
  # attribute will be set accordingly.
56
23
  #
57
- # @param [IntegrationDriver] driver
24
+ # @param [IntegrationTestRunner] runner
58
25
  # @return [true, false]
59
- def matches?(driver)
60
- self.driver = driver
61
- driver.run!
26
+ def matches?(runner)
27
+ self.runner = runner
28
+ runner.run!
62
29
  expectations.each do |expectation|
63
30
  unless output.send(match_function(expectation), expectation)
64
31
  self.failed_expectation = expectation
@@ -80,12 +47,12 @@ module Wukong
80
47
 
81
48
  # :nodoc:
82
49
  def failure_message
83
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
50
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
84
51
  end
85
52
 
86
53
  # :nodoc:
87
54
  def negative_failure_message
88
- "Expected #{output_description} of #{driver.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
55
+ "Expected #{output_description} of #{runner.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
89
56
  end
90
57
 
91
58
  # :nodoc:
@@ -95,18 +62,18 @@ module Wukong
95
62
 
96
63
  # :nodoc:
97
64
  def formatted_error_output
98
- output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{driver.stdout}" : "\n\nSTDERR was\n\n#{driver.stderr}"
65
+ output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{runner.stdout}" : "\n\nSTDERR was\n\n#{runner.stderr}"
99
66
  end
100
67
 
101
68
  # :nodoc:
102
69
  def formatted_command
103
- " $ #{driver.cmd}"
70
+ " $ #{runner.cmd}"
104
71
  end
105
72
 
106
73
  # :nodoc:
107
74
  def formatted_env
108
75
  [' {'].tap do |lines|
109
- driver.env.each_pair do |key, value|
76
+ runner.env.each_pair do |key, value|
110
77
  if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
111
78
  lines << " #{key} => #{value},"
112
79
  end
@@ -128,11 +95,11 @@ module Wukong
128
95
  end
129
96
 
130
97
  # A matcher for the STDOUT of a command.
131
- class StdoutMatcher < IntegrationMatcher
98
+ class StdoutMatcher < IntegrationTestMatcher
132
99
 
133
100
  # Picks the STDOUT of the command.
134
101
  def output
135
- driver.stdout
102
+ runner.stdout
136
103
  end
137
104
 
138
105
  # :nodoc:
@@ -147,11 +114,11 @@ module Wukong
147
114
  end
148
115
 
149
116
  # A matcher for the STDOUT of a command.
150
- class StderrMatcher < IntegrationMatcher
117
+ class StderrMatcher < IntegrationTestMatcher
151
118
 
152
119
  # Picks the STDOUT of the command.
153
120
  def output
154
- driver.stderr
121
+ runner.stderr
155
122
  end
156
123
 
157
124
  # :nodoc:
@@ -165,7 +132,7 @@ module Wukong
165
132
  end
166
133
 
167
134
  # A matcher for the exit code of a command.
168
- class ExitCodeMatcher < IntegrationMatcher
135
+ class ExitCodeMatcher < IntegrationTestMatcher
169
136
 
170
137
  # Initialize this matcher with the given `code`.
171
138
  #
@@ -184,27 +151,27 @@ module Wukong
184
151
  # Return whether or not the given command's exit code matches
185
152
  # the expectation.
186
153
  #
187
- # @param [IntegrationDriver] driver
154
+ # @param [IntegrationTestRunner] runner
188
155
  # @return [true, false]
189
- def matches?(driver)
190
- self.driver = driver
191
- driver.run!
156
+ def matches?(runner)
157
+ self.runner = runner
158
+ runner.run!
192
159
  if non_zero_exit_code?
193
- @failed = true if driver.exit_code == 0
160
+ @failed = true if runner.exit_code == 0
194
161
  else
195
- @failed = true if driver.exit_code != expected_exit_code
162
+ @failed = true if runner.exit_code != expected_exit_code
196
163
  end
197
164
  @failed ? false : true
198
165
  end
199
166
 
200
167
  # :nodoc:
201
168
  def failure_message
202
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{driver.exit_code} instead.#{formatted_error_output}"
169
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{runner.exit_code} instead.#{formatted_error_output}"
203
170
  end
204
171
 
205
172
  # :nodoc:
206
173
  def negative_failure_message
207
- "Ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
174
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
208
175
  end
209
176
 
210
177
  # :nodoc:
@@ -235,7 +202,6 @@ module Wukong
235
202
  def output_description
236
203
  "STDOUT"
237
204
  end
238
-
239
205
  end
240
206
  end
241
207
  end