ul-wukong 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (261) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +60 -0
  3. data/.gitmodules +6 -0
  4. data/.rspec +2 -0
  5. data/.travis.yml +19 -0
  6. data/.yardopts +6 -0
  7. data/CHANGELOG.md +7 -0
  8. data/Gemfile +17 -0
  9. data/Guardfile +12 -0
  10. data/LICENSE.md +95 -0
  11. data/NOTES-travis.md +31 -0
  12. data/README-old.md +422 -0
  13. data/README.md +1308 -0
  14. data/Rakefile +28 -0
  15. data/TODO.md +99 -0
  16. data/bin/cutc +30 -0
  17. data/bin/cuttab +5 -0
  18. data/bin/greptrue +6 -0
  19. data/bin/md5sort +20 -0
  20. data/bin/setcat +11 -0
  21. data/bin/tabchar +5 -0
  22. data/bin/uniq-ord +59 -0
  23. data/bin/uniqc +3 -0
  24. data/bin/wu +34 -0
  25. data/bin/wu-clean-encoding +31 -0
  26. data/bin/wu-date +13 -0
  27. data/bin/wu-datetime +13 -0
  28. data/bin/wu-hist +3 -0
  29. data/bin/wu-lign +186 -0
  30. data/bin/wu-local +4 -0
  31. data/bin/wu-plus +9 -0
  32. data/bin/wu-source +5 -0
  33. data/bin/wu-sum +31 -0
  34. data/diagrams/wu_local.dot +39 -0
  35. data/diagrams/wu_local.dot.png +0 -0
  36. data/examples/Gemfile +38 -0
  37. data/examples/README.md +9 -0
  38. data/examples/basic/string_reverser.rb +23 -0
  39. data/examples/basic/tiny_count.rb +8 -0
  40. data/examples/basic/word_count/accumulator.rb +26 -0
  41. data/examples/basic/word_count/tokenizer.rb +13 -0
  42. data/examples/basic/word_count/word_count.rb +6 -0
  43. data/examples/dataflow/scraper_macro_flow.rb +28 -0
  44. data/examples/deploy_pack/Gemfile +6 -0
  45. data/examples/deploy_pack/README.md +6 -0
  46. data/examples/deploy_pack/a/b/c/.gitkeep +0 -0
  47. data/examples/deploy_pack/app/processors/string_reverser.rb +5 -0
  48. data/examples/deploy_pack/config/environment.rb +1 -0
  49. data/examples/dsl/dataflow/fibonacci_series.rb +101 -0
  50. data/examples/dsl/dataflow/scraper_macro_flow.rb +28 -0
  51. data/examples/dsl/dataflow/simple.rb +12 -0
  52. data/examples/dsl/dataflow/telegram.rb +45 -0
  53. data/examples/dsl/workflow/cherry_pie.dot +97 -0
  54. data/examples/dsl/workflow/cherry_pie.md +104 -0
  55. data/examples/dsl/workflow/cherry_pie.png +0 -0
  56. data/examples/dsl/workflow/cherry_pie.rb +101 -0
  57. data/examples/empty/.gitkeep +0 -0
  58. data/examples/examples_helper.rb +9 -0
  59. data/examples/geo.rb +4 -0
  60. data/examples/geo/geo_grids.numbers +0 -0
  61. data/examples/geo/geolocated.rb +331 -0
  62. data/examples/geo/quadtile.rb +69 -0
  63. data/examples/geo/spec/geolocated_spec.rb +247 -0
  64. data/examples/geo/tile_fetcher.rb +77 -0
  65. data/examples/graph/implied_geolocation/README.md +63 -0
  66. data/examples/graph/minimum_spanning_tree/airfares_graphviz.rb +73 -0
  67. data/examples/improver/tweet_summary.rb +73 -0
  68. data/examples/loadable.rb +2 -0
  69. data/examples/munging/airline_flights/airline_flights.rake +83 -0
  70. data/examples/munging/airline_flights/airplane.rb +0 -0
  71. data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
  72. data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
  73. data/examples/munging/airline_flights/indexable.rb +75 -0
  74. data/examples/munging/airline_flights/indexable_spec.rb +90 -0
  75. data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
  76. data/examples/munging/airline_flights/tasks.rake +83 -0
  77. data/examples/munging/airline_flights/topcities.rb +167 -0
  78. data/examples/munging/geo/geo_json.rb +54 -0
  79. data/examples/munging/geo/geo_models.rb +69 -0
  80. data/examples/munging/geo/geonames_models.rb +107 -0
  81. data/examples/munging/geo/iso_codes.rb +172 -0
  82. data/examples/munging/geo/reconcile_countries.rb +124 -0
  83. data/examples/munging/geo/tasks.rake +71 -0
  84. data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
  85. data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
  86. data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
  87. data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
  88. data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
  89. data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
  90. data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
  91. data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +5 -0
  92. data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
  93. data/examples/munging/wikipedia/dbpedia/extract_links-cruft.rb +66 -0
  94. data/examples/munging/wikipedia/dbpedia/extract_links.rb +260 -0
  95. data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
  96. data/examples/rake_helper.rb +97 -0
  97. data/examples/ruby_project/Gemfile +6 -0
  98. data/examples/ruby_project/README.md +6 -0
  99. data/examples/ruby_project/a/b/c/.gitkeep +0 -0
  100. data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
  101. data/examples/server_logs/logline.rb +95 -0
  102. data/examples/server_logs/models.rb +66 -0
  103. data/examples/server_logs/page_counts.pig +48 -0
  104. data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
  105. data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
  106. data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
  107. data/examples/server_logs/server_logs-03-breadcrumbs-full.rb +71 -0
  108. data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
  109. data/examples/serverlogs/geo_ip_mapping/munge_geolite.rb +82 -0
  110. data/examples/serverlogs/models/logline.rb +102 -0
  111. data/examples/serverlogs/parser/apache_parser_widget.rb +46 -0
  112. data/examples/serverlogs/visit_paths/common.rb +4 -0
  113. data/examples/serverlogs/visit_paths/page_counts.pig +48 -0
  114. data/examples/serverlogs/visit_paths/serverlogs-01-parse-script.rb +11 -0
  115. data/examples/serverlogs/visit_paths/serverlogs-02-histograms-full.rb +31 -0
  116. data/examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb +12 -0
  117. data/examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb +67 -0
  118. data/examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb +38 -0
  119. data/examples/splitter.rb +94 -0
  120. data/examples/string_reverser.rb +7 -0
  121. data/examples/text/pig_latin/pig_latinizer.rb +35 -0
  122. data/examples/text/pig_latin/pig_latinizer_widget.rb +16 -0
  123. data/examples/text/regional_flavor/README.md +14 -0
  124. data/examples/text/regional_flavor/article_wordbags.pig +39 -0
  125. data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
  126. data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
  127. data/examples/twitter.rb +5 -0
  128. data/lib/hanuman.rb +36 -0
  129. data/lib/hanuman/graph.rb +97 -0
  130. data/lib/hanuman/graphvizzer.rb +206 -0
  131. data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
  132. data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
  133. data/lib/hanuman/link.rb +35 -0
  134. data/lib/hanuman/registry.rb +46 -0
  135. data/lib/hanuman/stage.rb +128 -0
  136. data/lib/hanuman/tree.rb +67 -0
  137. data/lib/wu/geo.rb +4 -0
  138. data/lib/wu/geo/geo_grids.numbers +0 -0
  139. data/lib/wu/geo/geolocated.rb +331 -0
  140. data/lib/wu/geo/quadtile.rb +69 -0
  141. data/lib/wu/graph/union_find.rb +62 -0
  142. data/lib/wu/model/reconcilable.rb +63 -0
  143. data/lib/wu/munging.rb +71 -0
  144. data/lib/wu/social/models/twitter.rb +31 -0
  145. data/lib/wu/wikipedia/models.rb +20 -0
  146. data/lib/wukong.rb +54 -0
  147. data/lib/wukong/dataflow.rb +43 -0
  148. data/lib/wukong/doc_helpers.rb +14 -0
  149. data/lib/wukong/doc_helpers/dataflow_handler.rb +29 -0
  150. data/lib/wukong/doc_helpers/field_handler.rb +91 -0
  151. data/lib/wukong/doc_helpers/processor_handler.rb +29 -0
  152. data/lib/wukong/driver.rb +214 -0
  153. data/lib/wukong/driver/event_machine_driver.rb +15 -0
  154. data/lib/wukong/driver/wiring.rb +68 -0
  155. data/lib/wukong/local.rb +42 -0
  156. data/lib/wukong/local/runner.rb +96 -0
  157. data/lib/wukong/local/stdio_driver.rb +104 -0
  158. data/lib/wukong/logger.rb +102 -0
  159. data/lib/wukong/model/faker.rb +136 -0
  160. data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
  161. data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
  162. data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
  163. data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
  164. data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
  165. data/lib/wukong/plugin.rb +48 -0
  166. data/lib/wukong/processor.rb +110 -0
  167. data/lib/wukong/rake_helper.rb +6 -0
  168. data/lib/wukong/runner.rb +169 -0
  169. data/lib/wukong/runner/boot_sequence.rb +123 -0
  170. data/lib/wukong/runner/code_loader.rb +52 -0
  171. data/lib/wukong/runner/command_runner.rb +44 -0
  172. data/lib/wukong/runner/deploy_pack_loader.rb +75 -0
  173. data/lib/wukong/runner/help_message.rb +42 -0
  174. data/lib/wukong/source.rb +33 -0
  175. data/lib/wukong/source/source_driver.rb +74 -0
  176. data/lib/wukong/source/source_runner.rb +38 -0
  177. data/lib/wukong/spec_helpers.rb +74 -0
  178. data/lib/wukong/spec_helpers/integration_tests.rb +150 -0
  179. data/lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb +207 -0
  180. data/lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb +97 -0
  181. data/lib/wukong/spec_helpers/shared_examples.rb +22 -0
  182. data/lib/wukong/spec_helpers/unit_tests.rb +135 -0
  183. data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +132 -0
  184. data/lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb +169 -0
  185. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +60 -0
  186. data/lib/wukong/version.rb +3 -0
  187. data/lib/wukong/widget/echo.rb +55 -0
  188. data/lib/wukong/widget/extract.rb +122 -0
  189. data/lib/wukong/widget/filters.rb +452 -0
  190. data/lib/wukong/widget/logger.rb +56 -0
  191. data/lib/wukong/widget/operators.rb +82 -0
  192. data/lib/wukong/widget/reducers.rb +10 -0
  193. data/lib/wukong/widget/reducers/accumulator.rb +73 -0
  194. data/lib/wukong/widget/reducers/bin.rb +368 -0
  195. data/lib/wukong/widget/reducers/count.rb +73 -0
  196. data/lib/wukong/widget/reducers/group.rb +128 -0
  197. data/lib/wukong/widget/reducers/group_concat.rb +98 -0
  198. data/lib/wukong/widget/reducers/improver.rb +71 -0
  199. data/lib/wukong/widget/reducers/join_xml.rb +37 -0
  200. data/lib/wukong/widget/reducers/moments.rb +72 -0
  201. data/lib/wukong/widget/reducers/sort.rb +180 -0
  202. data/lib/wukong/widget/reducers/uniq.rb +91 -0
  203. data/lib/wukong/widget/serializers.rb +317 -0
  204. data/lib/wukong/widget/utils.rb +46 -0
  205. data/lib/wukong/widgets.rb +7 -0
  206. data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
  207. data/spec/examples/dataflow/parse_apache_logs_spec.rb +8 -0
  208. data/spec/examples/dataflow/parsing_spec.rb +14 -0
  209. data/spec/examples/dataflow/simple_spec.rb +34 -0
  210. data/spec/examples/dataflow/telegram_spec.rb +43 -0
  211. data/spec/examples/graph/minimum_spanning_tree_spec.rb +34 -0
  212. data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
  213. data/spec/examples/munging/airline_flights_spec.rb +202 -0
  214. data/spec/examples/text/pig_latin_spec.rb +18 -0
  215. data/spec/examples/workflow/cherry_pie_spec.rb +36 -0
  216. data/spec/hanuman/graph_spec.rb +119 -0
  217. data/spec/hanuman/hanuman_spec.rb +10 -0
  218. data/spec/hanuman/registry_spec.rb +123 -0
  219. data/spec/hanuman/stage_spec.rb +81 -0
  220. data/spec/hanuman/tree_spec.rb +119 -0
  221. data/spec/spec.opts +1 -0
  222. data/spec/spec_helper.rb +43 -0
  223. data/spec/support/example_test_helpers.rb +95 -0
  224. data/spec/support/hanuman_test_helpers.rb +92 -0
  225. data/spec/support/integration_helper.rb +38 -0
  226. data/spec/support/model_test_helpers.rb +115 -0
  227. data/spec/support/shared_context_for_graphs.rb +57 -0
  228. data/spec/support/shared_context_for_reducers.rb +37 -0
  229. data/spec/support/shared_examples_for_builders.rb +94 -0
  230. data/spec/support/shared_examples_for_shortcuts.rb +57 -0
  231. data/spec/wu/model/reconcilable_spec.rb +152 -0
  232. data/spec/wukong/dataflow_spec.rb +87 -0
  233. data/spec/wukong/driver_spec.rb +154 -0
  234. data/spec/wukong/local/runner_spec.rb +29 -0
  235. data/spec/wukong/local/stdio_driver_spec.rb +73 -0
  236. data/spec/wukong/local_spec.rb +6 -0
  237. data/spec/wukong/logger_spec.rb +49 -0
  238. data/spec/wukong/model/faker_spec.rb +132 -0
  239. data/spec/wukong/processor_spec.rb +21 -0
  240. data/spec/wukong/runner_spec.rb +132 -0
  241. data/spec/wukong/source_spec.rb +6 -0
  242. data/spec/wukong/widget/extract_spec.rb +101 -0
  243. data/spec/wukong/widget/filters_spec.rb +79 -0
  244. data/spec/wukong/widget/logger_spec.rb +23 -0
  245. data/spec/wukong/widget/operators_spec.rb +25 -0
  246. data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
  247. data/spec/wukong/widget/reducers/count_spec.rb +11 -0
  248. data/spec/wukong/widget/reducers/group_spec.rb +21 -0
  249. data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
  250. data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
  251. data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
  252. data/spec/wukong/widget/reducers/uniq_spec.rb +14 -0
  253. data/spec/wukong/widget/serializers_spec.rb +114 -0
  254. data/spec/wukong/widget/sink_spec.rb +19 -0
  255. data/spec/wukong/widget/source_spec.rb +65 -0
  256. data/spec/wukong/wu-local_spec.rb +109 -0
  257. data/spec/wukong/wu-source_spec.rb +32 -0
  258. data/spec/wukong/wu_spec.rb +14 -0
  259. data/spec/wukong/wukong_spec.rb +10 -0
  260. data/wukong.gemspec +35 -0
  261. metadata +465 -0
@@ -0,0 +1,38 @@
1
+ require_relative('source_driver')
2
+ module Wukong
3
+ module Source
4
+
5
+ # Implements the `wu-source` command.
6
+ class SourceRunner < Wukong::Local::LocalRunner
7
+
8
+ usage "PROCESSOR|DATAFLOW"
9
+
10
+ description <<-EOF.gsub(/^ {8}/,'')
11
+ wu-source is a tool for using Wukong processors as sources of
12
+ data in streams.
13
+
14
+ Run any Wukong processor as a source for data:
15
+
16
+ $ wu-source fake_log_data
17
+ 205.4.75.208 - 3918471017 [27/Nov/2012:05:06:57 -0600] "GET /products/eget HTTP/1.0" 200 25600
18
+ 63.181.105.15 - 3650805763 [27/Nov/2012:05:06:57 -0600] "GET /products/lacinia-nulla-vitae HTTP/1.0" 200 3790
19
+ 227.190.78.101 - 39543891 [27/Nov/2012:05:06:58 -0600] "GET /products/odio-nulla-nulla-ipsum HTTP/1.0" 200 31718
20
+ ...
21
+
22
+ The fake_log_data processor will receive an event once every
23
+ second. Each event will consist of a single string giving a
24
+ consecutive integer starting with '1' as the first event.
25
+ EOF
26
+
27
+ include Logging
28
+
29
+ # The driver class used by `wu-source`.
30
+ #
31
+ # @return [Class]
32
+ def driver
33
+ SourceDriver
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,74 @@
1
+ require 'wukong'
2
+ require_relative('spec_helpers/unit_tests')
3
+ require_relative('spec_helpers/integration_tests')
4
+ require_relative('spec_helpers/shared_examples')
5
+
6
+ module Wukong
7
+
8
+ # This module lets you use write processor specs at a high level.
9
+ # Require it in your <tt>spec_helper.rb</tt> file:
10
+ #
11
+ # # in your spec/spec_helper.rb
12
+ # require 'wukong/spec_helpers'
13
+ # RSpec.configure do |config|
14
+ # include Wukong::SpecHelpers
15
+ # end
16
+ #
17
+ # Processors in a Wukong spec will have a collection of
18
+ # <tt>given_*</tt> methods you can use to (lazily) feed them records
19
+ # without having to have to build your own driver to run the
20
+ # processors.
21
+ #
22
+ # To each <tt>given_*</tt> method corresponds an <tt>emit_*</tt>
23
+ # matcher which will actually run the processor on the given
24
+ # inputs and compare against expected results. Here's an example,
25
+ # using a simple `tokenizer` processor.
26
+ #
27
+ # subject { processor(:tokenizer) }
28
+ #
29
+ # it "emits each word in a given string" do
30
+ # given("It was the best of times, it was the worst of times.").should emit(12).records
31
+ # end
32
+ #
33
+ # # Give similar input and check against explicit expected
34
+ # # output.
35
+ # it "should ignore punctuation and capitalization" do
36
+ # processor.given("You're crazy!").should emit("youre", "crazy")
37
+ # end
38
+ #
39
+ # # Pass the input but transform to JSON first (delimited and
40
+ # # as_tsv also work).
41
+ # it "should tokenize the 'text' attribute of a record if given JSON" do
42
+ # processor.given("text" => "Will be cast to JSON").as_json.should emit("will", "be", "cast", "to", "json")
43
+ # end
44
+ #
45
+ # # Initialize the :tokenizer processor with arguments to test
46
+ # # behavior under different conditions.
47
+ # it "should output a single record when asked for JSON output" do
48
+ # processor(:json => true).given("It was the best of times, it was the worst of times.").should emit(1).records
49
+ # end
50
+ #
51
+ # # Initialize processor with arguments and express that the
52
+ # # expected output will be in JSON though given as an object.
53
+ # it "should output all the tokens for its input record with its JSON output" do
54
+ # processor(:json => true).given("You're crazy!").should emit("tokens" => ["youre", "crazy"]).as_json
55
+ # end
56
+ #
57
+ # # Initialize processor with arguments, and both input and
58
+ # # output will be serialized/deserialized to/from JSON
59
+ # # automatically.
60
+ # it "can read and write pure JSON" do
61
+ # processor(:json => true).given("text" => "You're crazy!").as_json.should emit("tokens" => ["youre", "crazy"]).as_json
62
+ # end
63
+ #
64
+ # # Use a processor outside the scope of the top-level :describe
65
+ # # block.
66
+ # it "has a friend which does the same thing" do
67
+ # processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
68
+ # end
69
+ module SpecHelpers
70
+ include UnitTests
71
+ include IntegrationTests
72
+ end
73
+ end
74
+
@@ -0,0 +1,150 @@
1
+ require_relative('integration_tests/integration_test_runner')
2
+ require_relative('integration_tests/integration_test_matchers')
3
+
4
+ module Wukong
5
+ module SpecHelpers
6
+
7
+ # This module defines methods that are helpful to use in
8
+ # integration tests which require reading files from the local
9
+ # repository.
10
+ #
11
+ # Integration tests will spawn new system processes with their own
12
+ # environments. This module provides methods and hooks for
13
+ # customizing that environment.
14
+ module IntegrationTests
15
+
16
+ # The directory to add to the `RUBYLIB` environment variable for
17
+ # the spawned processes.
18
+ #
19
+ # If `args` are given, return a path within this directory.
20
+ #
21
+ # @param [Array<String>] args
22
+ # @return [String]
23
+ def lib_dir *args
24
+ root.join('lib', *args).to_s
25
+ end
26
+
27
+ # The directory to add to the `PATH` environment variable for
28
+ # the spawned processes.
29
+ #
30
+ # If `args` are given, return a path within this directory.
31
+ #
32
+ # @param [Array<String>] args
33
+ # @return [String]
34
+ def bin_dir *args
35
+ root.join('bin', *args).to_s
36
+ end
37
+
38
+ # The directory to use for examples for the spawned process.
39
+ #
40
+ # If `args` are given, return a path within this directory.
41
+ #
42
+ # @param [Array<String>] args
43
+ # @return [String]
44
+ def examples_dir *args
45
+ root.join('examples', *args).to_s
46
+ end
47
+
48
+ # A Hash of environment variables to use for the spawned
49
+ # process.
50
+ #
51
+ # By default, will put the IntegrationHelper#lib_dir on the
52
+ # `RUBYLIB` and the IntegrationHelper#bin_dir on the `PATH`.
53
+ #
54
+ # @return [Hash]
55
+ def integration_env
56
+ {
57
+ "PATH" => [bin_dir.to_s, ENV["PATH"]].compact.join(':'),
58
+ "RUBYLIB" => [lib_dir.to_s, ENV["RUBYLIB"]].compact.join(':')
59
+ }
60
+ end
61
+
62
+ # The directory to spawn new processes in.
63
+ #
64
+ # @return [String]
65
+ def integration_cwd
66
+ root.to_s
67
+ end
68
+
69
+ # Checks that each `expectation` appears in the STDOUT of the
70
+ # command. Order is irrelevant and each `expectation` can be
71
+ # either a String to check for inclusion or a Regexp to match
72
+ # with.
73
+ #
74
+ # @param [Array<String,Regexp>] expectations
75
+ def have_stdout *expectations
76
+ StdoutMatcher.new(*expectations)
77
+ end
78
+
79
+ # Checks that each `expectation` appears in the STDERR of the
80
+ # command. Order is irrelevant and each `expectation` can be
81
+ # either a String to check for inclusion or a Regexp to match
82
+ # with.
83
+ #
84
+ # @param [Array<String,Regexp>] expectations
85
+ def have_stderr *expectations
86
+ StderrMatcher.new(*expectations)
87
+ end
88
+
89
+ # Checks that the command exits with the given `code`.
90
+ #
91
+ # @param [Integer] code
92
+ def exit_with code
93
+ ExitCodeMatcher.new(code)
94
+ end
95
+
96
+ # Spawn a command and capture its STDOUT, STDERR, and exit code.
97
+ #
98
+ # The `args` will be joined together into a command line.
99
+ #
100
+ # It is expected that you will use the matchers defined in
101
+ # IntegrationMatchers in your integration tests:
102
+ #
103
+ # @example Check output of 'ls' includes a string 'foo.txt'
104
+ # it "lists files" do
105
+ # command('ls').should have_output('foo.txt')
106
+ # end
107
+ #
108
+ # @example More complicated
109
+ # context "long format" do
110
+ # it "lists files with timestamps" do
111
+ # command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
112
+ # end
113
+ # end
114
+ #
115
+ # @param [Array<String>] args
116
+ #
117
+ # @overload command(*args, options={})
118
+ # If the last element of `args` is a Hash it will be used for
119
+ # options.
120
+ #
121
+ # The :env option specifies the command line environment to
122
+ # use for the command. By default this will be the value of
123
+ # the Ruby process's own `ENV` variable. If running in a
124
+ # context in which the `integration_env` method is defined,
125
+ # its return value will be merged on top of `ENV`. An
126
+ # explicitly provided :env option will again be merged on top.
127
+ #
128
+ # The :cwd option specifies the working directory to start in.
129
+ # It defaults to the value of <tt>Dir.pwd</tt>
130
+ #
131
+ # @param [Array<String>] args
132
+ # @param [Hash] options
133
+ # @option options [Hash] env the shell environment to spawn the command with
134
+ # @option options [Hash] cwd the directory to execute the command in
135
+ def command *args
136
+ a = args.flatten.compact
137
+ options = (a.last.is_a?(Hash) ? a.pop : {})
138
+
139
+ env = ENV.to_hash.dup
140
+ env.merge!(integration_env) if respond_to?(:integration_env)
141
+ env.merge!(options[:env] || {})
142
+
143
+ cwd = options[:cwd]
144
+ cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
145
+
146
+ IntegrationTestRunner.new(a, cwd: cwd, env: env)
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,207 @@
1
+ module Wukong
2
+ module SpecHelpers
3
+
4
+ # A class for running commands and capturing their STDOUT, STDERR,
5
+ # and exit code. This class is designed to work with the matchers
6
+ # defined in IntegrationTestMatchers.
7
+ class IntegrationTestMatcher
8
+
9
+ # The runner used to run the actual commands.
10
+ attr_accessor :runner
11
+
12
+ # An array of expectations about the output of the runner.
13
+ attr_accessor :expectations
14
+
15
+ # The expectation which caused failure.
16
+ attr_accessor :failed_expectation
17
+
18
+ # Return whether or not the given command's output matches
19
+ # expectations.
20
+ #
21
+ # If an expectation failes to match, the `failed_expectation`
22
+ # attribute will be set accordingly.
23
+ #
24
+ # @param [IntegrationTestRunner] runner
25
+ # @return [true, false]
26
+ def matches?(runner)
27
+ self.runner = runner
28
+ runner.run!
29
+ expectations.each do |expectation|
30
+ unless output.send(match_function(expectation), expectation)
31
+ self.failed_expectation = expectation
32
+ return false
33
+ end
34
+ end
35
+ true
36
+ end
37
+
38
+ # Create a matcher on the given expectations. Each expectation
39
+ # can be either a String or a Regexp. Strings will be tested
40
+ # for inclusion in the output, Regexps will be tested for a
41
+ # match against the output.
42
+ #
43
+ # @param [Array<String,Regexp>] expectations
44
+ def initialize *expectations
45
+ self.expectations = expectations
46
+ end
47
+
48
+ # :nodoc:
49
+ def failure_message
50
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
51
+ end
52
+
53
+ # :nodoc:
54
+ def negative_failure_message
55
+ "Expected #{output_description} of #{runner.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
56
+ end
57
+
58
+ # :nodoc:
59
+ def formatted_output
60
+ output.split("\n").map { |line| ' ' + line }.join("\n")
61
+ end
62
+
63
+ # :nodoc:
64
+ def formatted_error_output
65
+ output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{runner.stdout}" : "\n\nSTDERR was\n\n#{runner.stderr}"
66
+ end
67
+
68
+ # :nodoc:
69
+ def formatted_command
70
+ " $ #{runner.cmd}"
71
+ end
72
+
73
+ # :nodoc:
74
+ def formatted_env
75
+ [' {'].tap do |lines|
76
+ runner.env.each_pair do |key, value|
77
+ if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
78
+ lines << " #{key} => #{value},"
79
+ end
80
+ end
81
+ lines << ' }'
82
+ end.join("\n")
83
+ end
84
+
85
+ # :nodoc:
86
+ def match_function expectation
87
+ expectation.is_a?(Regexp) ? :match : :include?
88
+ end
89
+
90
+ # :nodoc:
91
+ def match_type
92
+ failed_expectation.is_a?(Regexp) ? 'match' : 'include'
93
+ end
94
+
95
+ end
96
+
97
+ # A matcher for the STDOUT of a command.
98
+ class StdoutMatcher < IntegrationTestMatcher
99
+
100
+ # Picks the STDOUT of the command.
101
+ def output
102
+ runner.stdout
103
+ end
104
+
105
+ # :nodoc:
106
+ def output_description
107
+ "STDOUT"
108
+ end
109
+
110
+ def description
111
+ "have the correct #{output_description}"
112
+ end
113
+
114
+ end
115
+
116
+ # A matcher for the STDOUT of a command.
117
+ class StderrMatcher < IntegrationTestMatcher
118
+
119
+ # Picks the STDOUT of the command.
120
+ def output
121
+ runner.stderr
122
+ end
123
+
124
+ # :nodoc:
125
+ def output_description
126
+ "STDERR"
127
+ end
128
+
129
+ def description
130
+ "print an appropriate error message on #{output_description}"
131
+ end
132
+ end
133
+
134
+ # A matcher for the exit code of a command.
135
+ class ExitCodeMatcher < IntegrationTestMatcher
136
+
137
+ # Initialize this matcher with the given `code`.
138
+ #
139
+ # If `code` is the symbol <tt>:non_zero</tt> then the
140
+ # expectation will be any non-zero exit code.
141
+ #
142
+ # @param [Integer,Symbol] code
143
+ def initialize code
144
+ if code == :non_zero
145
+ @expected_code = :non_zero
146
+ else
147
+ @expected_code = code.to_i
148
+ end
149
+ end
150
+
151
+ # Return whether or not the given command's exit code matches
152
+ # the expectation.
153
+ #
154
+ # @param [IntegrationTestRunner] runner
155
+ # @return [true, false]
156
+ def matches?(runner)
157
+ self.runner = runner
158
+ runner.run!
159
+ if non_zero_exit_code?
160
+ @failed = true if runner.exit_code == 0
161
+ else
162
+ @failed = true if runner.exit_code != expected_exit_code
163
+ end
164
+ @failed ? false : true
165
+ end
166
+
167
+ # :nodoc:
168
+ def failure_message
169
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{runner.exit_code} instead.#{formatted_error_output}"
170
+ end
171
+
172
+ # :nodoc:
173
+ def negative_failure_message
174
+ "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
175
+ end
176
+
177
+ # :nodoc:
178
+ def non_zero_exit_code?
179
+ @expected_code == :non_zero
180
+ end
181
+
182
+ # :nodoc:
183
+ def expected_exit_code
184
+ (@expected_code || 0).to_i
185
+ end
186
+
187
+ # :nodoc:
188
+ def expected_exit_code_description
189
+ if non_zero_exit_code?
190
+ "a non-zero exit code"
191
+ else
192
+ "an exit code of #{expected_exit_code}"
193
+ end
194
+ end
195
+
196
+ # :nodoc:
197
+ def description
198
+ "exit with #{expected_exit_code_description}"
199
+ end
200
+
201
+ # :nodoc:
202
+ def output_description
203
+ "STDOUT"
204
+ end
205
+ end
206
+ end
207
+ end