ul-wukong 4.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +60 -0
- data/.gitmodules +6 -0
- data/.rspec +2 -0
- data/.travis.yml +19 -0
- data/.yardopts +6 -0
- data/CHANGELOG.md +7 -0
- data/Gemfile +17 -0
- data/Guardfile +12 -0
- data/LICENSE.md +95 -0
- data/NOTES-travis.md +31 -0
- data/README-old.md +422 -0
- data/README.md +1308 -0
- data/Rakefile +28 -0
- data/TODO.md +99 -0
- data/bin/cutc +30 -0
- data/bin/cuttab +5 -0
- data/bin/greptrue +6 -0
- data/bin/md5sort +20 -0
- data/bin/setcat +11 -0
- data/bin/tabchar +5 -0
- data/bin/uniq-ord +59 -0
- data/bin/uniqc +3 -0
- data/bin/wu +34 -0
- data/bin/wu-clean-encoding +31 -0
- data/bin/wu-date +13 -0
- data/bin/wu-datetime +13 -0
- data/bin/wu-hist +3 -0
- data/bin/wu-lign +186 -0
- data/bin/wu-local +4 -0
- data/bin/wu-plus +9 -0
- data/bin/wu-source +5 -0
- data/bin/wu-sum +31 -0
- data/diagrams/wu_local.dot +39 -0
- data/diagrams/wu_local.dot.png +0 -0
- data/examples/Gemfile +38 -0
- data/examples/README.md +9 -0
- data/examples/basic/string_reverser.rb +23 -0
- data/examples/basic/tiny_count.rb +8 -0
- data/examples/basic/word_count/accumulator.rb +26 -0
- data/examples/basic/word_count/tokenizer.rb +13 -0
- data/examples/basic/word_count/word_count.rb +6 -0
- data/examples/dataflow/scraper_macro_flow.rb +28 -0
- data/examples/deploy_pack/Gemfile +6 -0
- data/examples/deploy_pack/README.md +6 -0
- data/examples/deploy_pack/a/b/c/.gitkeep +0 -0
- data/examples/deploy_pack/app/processors/string_reverser.rb +5 -0
- data/examples/deploy_pack/config/environment.rb +1 -0
- data/examples/dsl/dataflow/fibonacci_series.rb +101 -0
- data/examples/dsl/dataflow/scraper_macro_flow.rb +28 -0
- data/examples/dsl/dataflow/simple.rb +12 -0
- data/examples/dsl/dataflow/telegram.rb +45 -0
- data/examples/dsl/workflow/cherry_pie.dot +97 -0
- data/examples/dsl/workflow/cherry_pie.md +104 -0
- data/examples/dsl/workflow/cherry_pie.png +0 -0
- data/examples/dsl/workflow/cherry_pie.rb +101 -0
- data/examples/empty/.gitkeep +0 -0
- data/examples/examples_helper.rb +9 -0
- data/examples/geo.rb +4 -0
- data/examples/geo/geo_grids.numbers +0 -0
- data/examples/geo/geolocated.rb +331 -0
- data/examples/geo/quadtile.rb +69 -0
- data/examples/geo/spec/geolocated_spec.rb +247 -0
- data/examples/geo/tile_fetcher.rb +77 -0
- data/examples/graph/implied_geolocation/README.md +63 -0
- data/examples/graph/minimum_spanning_tree/airfares_graphviz.rb +73 -0
- data/examples/improver/tweet_summary.rb +73 -0
- data/examples/loadable.rb +2 -0
- data/examples/munging/airline_flights/airline_flights.rake +83 -0
- data/examples/munging/airline_flights/airplane.rb +0 -0
- data/examples/munging/airline_flights/airport_id_unification.rb +129 -0
- data/examples/munging/airline_flights/airport_ok_chars.rb +4 -0
- data/examples/munging/airline_flights/indexable.rb +75 -0
- data/examples/munging/airline_flights/indexable_spec.rb +90 -0
- data/examples/munging/airline_flights/reconcile_airports.rb +142 -0
- data/examples/munging/airline_flights/tasks.rake +83 -0
- data/examples/munging/airline_flights/topcities.rb +167 -0
- data/examples/munging/geo/geo_json.rb +54 -0
- data/examples/munging/geo/geo_models.rb +69 -0
- data/examples/munging/geo/geonames_models.rb +107 -0
- data/examples/munging/geo/iso_codes.rb +172 -0
- data/examples/munging/geo/reconcile_countries.rb +124 -0
- data/examples/munging/geo/tasks.rake +71 -0
- data/examples/munging/wikipedia/articles/extract_articles-parsed.rb +79 -0
- data/examples/munging/wikipedia/articles/extract_articles-templated.rb +136 -0
- data/examples/munging/wikipedia/articles/textualize_articles.rb +54 -0
- data/examples/munging/wikipedia/articles/verify_structure.rb +43 -0
- data/examples/munging/wikipedia/articles/wp2txt-LICENSE.txt +22 -0
- data/examples/munging/wikipedia/articles/wp2txt_article.rb +259 -0
- data/examples/munging/wikipedia/articles/wp2txt_utils.rb +452 -0
- data/examples/munging/wikipedia/dbpedia/dbpedia_common.rb +5 -0
- data/examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb +78 -0
- data/examples/munging/wikipedia/dbpedia/extract_links-cruft.rb +66 -0
- data/examples/munging/wikipedia/dbpedia/extract_links.rb +260 -0
- data/examples/munging/wikipedia/dbpedia/sameas_extractor.rb +20 -0
- data/examples/rake_helper.rb +97 -0
- data/examples/ruby_project/Gemfile +6 -0
- data/examples/ruby_project/README.md +6 -0
- data/examples/ruby_project/a/b/c/.gitkeep +0 -0
- data/examples/server_logs/geo_ip_mapping/munge_geolite.rb +82 -0
- data/examples/server_logs/logline.rb +95 -0
- data/examples/server_logs/models.rb +66 -0
- data/examples/server_logs/page_counts.pig +48 -0
- data/examples/server_logs/server_logs-01-parse-script.rb +13 -0
- data/examples/server_logs/server_logs-02-histograms-full.rb +33 -0
- data/examples/server_logs/server_logs-02-histograms-mapper.rb +14 -0
- data/examples/server_logs/server_logs-03-breadcrumbs-full.rb +71 -0
- data/examples/server_logs/server_logs-04-page_page_edges-full.rb +40 -0
- data/examples/serverlogs/geo_ip_mapping/munge_geolite.rb +82 -0
- data/examples/serverlogs/models/logline.rb +102 -0
- data/examples/serverlogs/parser/apache_parser_widget.rb +46 -0
- data/examples/serverlogs/visit_paths/common.rb +4 -0
- data/examples/serverlogs/visit_paths/page_counts.pig +48 -0
- data/examples/serverlogs/visit_paths/serverlogs-01-parse-script.rb +11 -0
- data/examples/serverlogs/visit_paths/serverlogs-02-histograms-full.rb +31 -0
- data/examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb +12 -0
- data/examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb +67 -0
- data/examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb +38 -0
- data/examples/splitter.rb +94 -0
- data/examples/string_reverser.rb +7 -0
- data/examples/text/pig_latin/pig_latinizer.rb +35 -0
- data/examples/text/pig_latin/pig_latinizer_widget.rb +16 -0
- data/examples/text/regional_flavor/README.md +14 -0
- data/examples/text/regional_flavor/article_wordbags.pig +39 -0
- data/examples/text/regional_flavor/j01-article_wordbags.rb +4 -0
- data/examples/text/regional_flavor/simple_pig_script.pig +27 -0
- data/examples/twitter.rb +5 -0
- data/lib/hanuman.rb +36 -0
- data/lib/hanuman/graph.rb +97 -0
- data/lib/hanuman/graphvizzer.rb +206 -0
- data/lib/hanuman/graphvizzer/gv_models.rb +161 -0
- data/lib/hanuman/graphvizzer/gv_presenter.rb +97 -0
- data/lib/hanuman/link.rb +35 -0
- data/lib/hanuman/registry.rb +46 -0
- data/lib/hanuman/stage.rb +128 -0
- data/lib/hanuman/tree.rb +67 -0
- data/lib/wu/geo.rb +4 -0
- data/lib/wu/geo/geo_grids.numbers +0 -0
- data/lib/wu/geo/geolocated.rb +331 -0
- data/lib/wu/geo/quadtile.rb +69 -0
- data/lib/wu/graph/union_find.rb +62 -0
- data/lib/wu/model/reconcilable.rb +63 -0
- data/lib/wu/munging.rb +71 -0
- data/lib/wu/social/models/twitter.rb +31 -0
- data/lib/wu/wikipedia/models.rb +20 -0
- data/lib/wukong.rb +54 -0
- data/lib/wukong/dataflow.rb +43 -0
- data/lib/wukong/doc_helpers.rb +14 -0
- data/lib/wukong/doc_helpers/dataflow_handler.rb +29 -0
- data/lib/wukong/doc_helpers/field_handler.rb +91 -0
- data/lib/wukong/doc_helpers/processor_handler.rb +29 -0
- data/lib/wukong/driver.rb +214 -0
- data/lib/wukong/driver/event_machine_driver.rb +15 -0
- data/lib/wukong/driver/wiring.rb +68 -0
- data/lib/wukong/local.rb +42 -0
- data/lib/wukong/local/runner.rb +96 -0
- data/lib/wukong/local/stdio_driver.rb +104 -0
- data/lib/wukong/logger.rb +102 -0
- data/lib/wukong/model/faker.rb +136 -0
- data/lib/wukong/model/flatpack_parser/flat.rb +60 -0
- data/lib/wukong/model/flatpack_parser/flatpack.rb +4 -0
- data/lib/wukong/model/flatpack_parser/lang.rb +46 -0
- data/lib/wukong/model/flatpack_parser/parser.rb +55 -0
- data/lib/wukong/model/flatpack_parser/tokens.rb +130 -0
- data/lib/wukong/plugin.rb +48 -0
- data/lib/wukong/processor.rb +110 -0
- data/lib/wukong/rake_helper.rb +6 -0
- data/lib/wukong/runner.rb +169 -0
- data/lib/wukong/runner/boot_sequence.rb +123 -0
- data/lib/wukong/runner/code_loader.rb +52 -0
- data/lib/wukong/runner/command_runner.rb +44 -0
- data/lib/wukong/runner/deploy_pack_loader.rb +75 -0
- data/lib/wukong/runner/help_message.rb +42 -0
- data/lib/wukong/source.rb +33 -0
- data/lib/wukong/source/source_driver.rb +74 -0
- data/lib/wukong/source/source_runner.rb +38 -0
- data/lib/wukong/spec_helpers.rb +74 -0
- data/lib/wukong/spec_helpers/integration_tests.rb +150 -0
- data/lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb +207 -0
- data/lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb +97 -0
- data/lib/wukong/spec_helpers/shared_examples.rb +22 -0
- data/lib/wukong/spec_helpers/unit_tests.rb +135 -0
- data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +132 -0
- data/lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb +169 -0
- data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +60 -0
- data/lib/wukong/version.rb +3 -0
- data/lib/wukong/widget/echo.rb +55 -0
- data/lib/wukong/widget/extract.rb +122 -0
- data/lib/wukong/widget/filters.rb +452 -0
- data/lib/wukong/widget/logger.rb +56 -0
- data/lib/wukong/widget/operators.rb +82 -0
- data/lib/wukong/widget/reducers.rb +10 -0
- data/lib/wukong/widget/reducers/accumulator.rb +73 -0
- data/lib/wukong/widget/reducers/bin.rb +368 -0
- data/lib/wukong/widget/reducers/count.rb +73 -0
- data/lib/wukong/widget/reducers/group.rb +128 -0
- data/lib/wukong/widget/reducers/group_concat.rb +98 -0
- data/lib/wukong/widget/reducers/improver.rb +71 -0
- data/lib/wukong/widget/reducers/join_xml.rb +37 -0
- data/lib/wukong/widget/reducers/moments.rb +72 -0
- data/lib/wukong/widget/reducers/sort.rb +180 -0
- data/lib/wukong/widget/reducers/uniq.rb +91 -0
- data/lib/wukong/widget/serializers.rb +317 -0
- data/lib/wukong/widget/utils.rb +46 -0
- data/lib/wukong/widgets.rb +7 -0
- data/spec/examples/dataflow/fibonacci_series_spec.rb +18 -0
- data/spec/examples/dataflow/parse_apache_logs_spec.rb +8 -0
- data/spec/examples/dataflow/parsing_spec.rb +14 -0
- data/spec/examples/dataflow/simple_spec.rb +34 -0
- data/spec/examples/dataflow/telegram_spec.rb +43 -0
- data/spec/examples/graph/minimum_spanning_tree_spec.rb +34 -0
- data/spec/examples/munging/airline_flights/identifiers_spec.rb +16 -0
- data/spec/examples/munging/airline_flights_spec.rb +202 -0
- data/spec/examples/text/pig_latin_spec.rb +18 -0
- data/spec/examples/workflow/cherry_pie_spec.rb +36 -0
- data/spec/hanuman/graph_spec.rb +119 -0
- data/spec/hanuman/hanuman_spec.rb +10 -0
- data/spec/hanuman/registry_spec.rb +123 -0
- data/spec/hanuman/stage_spec.rb +81 -0
- data/spec/hanuman/tree_spec.rb +119 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +43 -0
- data/spec/support/example_test_helpers.rb +95 -0
- data/spec/support/hanuman_test_helpers.rb +92 -0
- data/spec/support/integration_helper.rb +38 -0
- data/spec/support/model_test_helpers.rb +115 -0
- data/spec/support/shared_context_for_graphs.rb +57 -0
- data/spec/support/shared_context_for_reducers.rb +37 -0
- data/spec/support/shared_examples_for_builders.rb +94 -0
- data/spec/support/shared_examples_for_shortcuts.rb +57 -0
- data/spec/wu/model/reconcilable_spec.rb +152 -0
- data/spec/wukong/dataflow_spec.rb +87 -0
- data/spec/wukong/driver_spec.rb +154 -0
- data/spec/wukong/local/runner_spec.rb +29 -0
- data/spec/wukong/local/stdio_driver_spec.rb +73 -0
- data/spec/wukong/local_spec.rb +6 -0
- data/spec/wukong/logger_spec.rb +49 -0
- data/spec/wukong/model/faker_spec.rb +132 -0
- data/spec/wukong/processor_spec.rb +21 -0
- data/spec/wukong/runner_spec.rb +132 -0
- data/spec/wukong/source_spec.rb +6 -0
- data/spec/wukong/widget/extract_spec.rb +101 -0
- data/spec/wukong/widget/filters_spec.rb +79 -0
- data/spec/wukong/widget/logger_spec.rb +23 -0
- data/spec/wukong/widget/operators_spec.rb +25 -0
- data/spec/wukong/widget/reducers/bin_spec.rb +92 -0
- data/spec/wukong/widget/reducers/count_spec.rb +11 -0
- data/spec/wukong/widget/reducers/group_spec.rb +21 -0
- data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
- data/spec/wukong/widget/reducers/moments_spec.rb +36 -0
- data/spec/wukong/widget/reducers/sort_spec.rb +26 -0
- data/spec/wukong/widget/reducers/uniq_spec.rb +14 -0
- data/spec/wukong/widget/serializers_spec.rb +114 -0
- data/spec/wukong/widget/sink_spec.rb +19 -0
- data/spec/wukong/widget/source_spec.rb +65 -0
- data/spec/wukong/wu-local_spec.rb +109 -0
- data/spec/wukong/wu-source_spec.rb +32 -0
- data/spec/wukong/wu_spec.rb +14 -0
- data/spec/wukong/wukong_spec.rb +10 -0
- data/wukong.gemspec +35 -0
- metadata +465 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
require_relative('source_driver')
|
2
|
+
module Wukong
|
3
|
+
module Source
|
4
|
+
|
5
|
+
# Implements the `wu-source` command.
|
6
|
+
class SourceRunner < Wukong::Local::LocalRunner
|
7
|
+
|
8
|
+
usage "PROCESSOR|DATAFLOW"
|
9
|
+
|
10
|
+
description <<-EOF.gsub(/^ {8}/,'')
|
11
|
+
wu-source is a tool for using Wukong processors as sources of
|
12
|
+
data in streams.
|
13
|
+
|
14
|
+
Run any Wukong processor as a source for data:
|
15
|
+
|
16
|
+
$ wu-source fake_log_data
|
17
|
+
205.4.75.208 - 3918471017 [27/Nov/2012:05:06:57 -0600] "GET /products/eget HTTP/1.0" 200 25600
|
18
|
+
63.181.105.15 - 3650805763 [27/Nov/2012:05:06:57 -0600] "GET /products/lacinia-nulla-vitae HTTP/1.0" 200 3790
|
19
|
+
227.190.78.101 - 39543891 [27/Nov/2012:05:06:58 -0600] "GET /products/odio-nulla-nulla-ipsum HTTP/1.0" 200 31718
|
20
|
+
...
|
21
|
+
|
22
|
+
The fake_log_data processor will receive an event once every
|
23
|
+
second. Each event will consist of a single string giving a
|
24
|
+
consecutive integer starting with '1' as the first event.
|
25
|
+
EOF
|
26
|
+
|
27
|
+
include Logging
|
28
|
+
|
29
|
+
# The driver class used by `wu-source`.
|
30
|
+
#
|
31
|
+
# @return [Class]
|
32
|
+
def driver
|
33
|
+
SourceDriver
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'wukong'
|
2
|
+
require_relative('spec_helpers/unit_tests')
|
3
|
+
require_relative('spec_helpers/integration_tests')
|
4
|
+
require_relative('spec_helpers/shared_examples')
|
5
|
+
|
6
|
+
module Wukong
|
7
|
+
|
8
|
+
# This module lets you use write processor specs at a high level.
|
9
|
+
# Require it in your <tt>spec_helper.rb</tt> file:
|
10
|
+
#
|
11
|
+
# # in your spec/spec_helper.rb
|
12
|
+
# require 'wukong/spec_helpers'
|
13
|
+
# RSpec.configure do |config|
|
14
|
+
# include Wukong::SpecHelpers
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# Processors in a Wukong spec will have a collection of
|
18
|
+
# <tt>given_*</tt> methods you can use to (lazily) feed them records
|
19
|
+
# without having to have to build your own driver to run the
|
20
|
+
# processors.
|
21
|
+
#
|
22
|
+
# To each <tt>given_*</tt> method corresponds an <tt>emit_*</tt>
|
23
|
+
# matcher which will actually run the processor on the given
|
24
|
+
# inputs and compare against expected results. Here's an example,
|
25
|
+
# using a simple `tokenizer` processor.
|
26
|
+
#
|
27
|
+
# subject { processor(:tokenizer) }
|
28
|
+
#
|
29
|
+
# it "emits each word in a given string" do
|
30
|
+
# given("It was the best of times, it was the worst of times.").should emit(12).records
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# # Give similar input and check against explicit expected
|
34
|
+
# # output.
|
35
|
+
# it "should ignore punctuation and capitalization" do
|
36
|
+
# processor.given("You're crazy!").should emit("youre", "crazy")
|
37
|
+
# end
|
38
|
+
#
|
39
|
+
# # Pass the input but transform to JSON first (delimited and
|
40
|
+
# # as_tsv also work).
|
41
|
+
# it "should tokenize the 'text' attribute of a record if given JSON" do
|
42
|
+
# processor.given("text" => "Will be cast to JSON").as_json.should emit("will", "be", "cast", "to", "json")
|
43
|
+
# end
|
44
|
+
#
|
45
|
+
# # Initialize the :tokenizer processor with arguments to test
|
46
|
+
# # behavior under different conditions.
|
47
|
+
# it "should output a single record when asked for JSON output" do
|
48
|
+
# processor(:json => true).given("It was the best of times, it was the worst of times.").should emit(1).records
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
# # Initialize processor with arguments and express that the
|
52
|
+
# # expected output will be in JSON though given as an object.
|
53
|
+
# it "should output all the tokens for its input record with its JSON output" do
|
54
|
+
# processor(:json => true).given("You're crazy!").should emit("tokens" => ["youre", "crazy"]).as_json
|
55
|
+
# end
|
56
|
+
#
|
57
|
+
# # Initialize processor with arguments, and both input and
|
58
|
+
# # output will be serialized/deserialized to/from JSON
|
59
|
+
# # automatically.
|
60
|
+
# it "can read and write pure JSON" do
|
61
|
+
# processor(:json => true).given("text" => "You're crazy!").as_json.should emit("tokens" => ["youre", "crazy"]).as_json
|
62
|
+
# end
|
63
|
+
#
|
64
|
+
# # Use a processor outside the scope of the top-level :describe
|
65
|
+
# # block.
|
66
|
+
# it "has a friend which does the same thing" do
|
67
|
+
# processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
|
68
|
+
# end
|
69
|
+
module SpecHelpers
|
70
|
+
include UnitTests
|
71
|
+
include IntegrationTests
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
@@ -0,0 +1,150 @@
|
|
1
|
+
require_relative('integration_tests/integration_test_runner')
|
2
|
+
require_relative('integration_tests/integration_test_matchers')
|
3
|
+
|
4
|
+
module Wukong
|
5
|
+
module SpecHelpers
|
6
|
+
|
7
|
+
# This module defines methods that are helpful to use in
|
8
|
+
# integration tests which require reading files from the local
|
9
|
+
# repository.
|
10
|
+
#
|
11
|
+
# Integration tests will spawn new system processes with their own
|
12
|
+
# environments. This module provides methods and hooks for
|
13
|
+
# customizing that environment.
|
14
|
+
module IntegrationTests
|
15
|
+
|
16
|
+
# The directory to add to the `RUBYLIB` environment variable for
|
17
|
+
# the spawned processes.
|
18
|
+
#
|
19
|
+
# If `args` are given, return a path within this directory.
|
20
|
+
#
|
21
|
+
# @param [Array<String>] args
|
22
|
+
# @return [String]
|
23
|
+
def lib_dir *args
|
24
|
+
root.join('lib', *args).to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
# The directory to add to the `PATH` environment variable for
|
28
|
+
# the spawned processes.
|
29
|
+
#
|
30
|
+
# If `args` are given, return a path within this directory.
|
31
|
+
#
|
32
|
+
# @param [Array<String>] args
|
33
|
+
# @return [String]
|
34
|
+
def bin_dir *args
|
35
|
+
root.join('bin', *args).to_s
|
36
|
+
end
|
37
|
+
|
38
|
+
# The directory to use for examples for the spawned process.
|
39
|
+
#
|
40
|
+
# If `args` are given, return a path within this directory.
|
41
|
+
#
|
42
|
+
# @param [Array<String>] args
|
43
|
+
# @return [String]
|
44
|
+
def examples_dir *args
|
45
|
+
root.join('examples', *args).to_s
|
46
|
+
end
|
47
|
+
|
48
|
+
# A Hash of environment variables to use for the spawned
|
49
|
+
# process.
|
50
|
+
#
|
51
|
+
# By default, will put the IntegrationHelper#lib_dir on the
|
52
|
+
# `RUBYLIB` and the IntegrationHelper#bin_dir on the `PATH`.
|
53
|
+
#
|
54
|
+
# @return [Hash]
|
55
|
+
def integration_env
|
56
|
+
{
|
57
|
+
"PATH" => [bin_dir.to_s, ENV["PATH"]].compact.join(':'),
|
58
|
+
"RUBYLIB" => [lib_dir.to_s, ENV["RUBYLIB"]].compact.join(':')
|
59
|
+
}
|
60
|
+
end
|
61
|
+
|
62
|
+
# The directory to spawn new processes in.
|
63
|
+
#
|
64
|
+
# @return [String]
|
65
|
+
def integration_cwd
|
66
|
+
root.to_s
|
67
|
+
end
|
68
|
+
|
69
|
+
# Checks that each `expectation` appears in the STDOUT of the
|
70
|
+
# command. Order is irrelevant and each `expectation` can be
|
71
|
+
# either a String to check for inclusion or a Regexp to match
|
72
|
+
# with.
|
73
|
+
#
|
74
|
+
# @param [Array<String,Regexp>] expectations
|
75
|
+
def have_stdout *expectations
|
76
|
+
StdoutMatcher.new(*expectations)
|
77
|
+
end
|
78
|
+
|
79
|
+
# Checks that each `expectation` appears in the STDERR of the
|
80
|
+
# command. Order is irrelevant and each `expectation` can be
|
81
|
+
# either a String to check for inclusion or a Regexp to match
|
82
|
+
# with.
|
83
|
+
#
|
84
|
+
# @param [Array<String,Regexp>] expectations
|
85
|
+
def have_stderr *expectations
|
86
|
+
StderrMatcher.new(*expectations)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Checks that the command exits with the given `code`.
|
90
|
+
#
|
91
|
+
# @param [Integer] code
|
92
|
+
def exit_with code
|
93
|
+
ExitCodeMatcher.new(code)
|
94
|
+
end
|
95
|
+
|
96
|
+
# Spawn a command and capture its STDOUT, STDERR, and exit code.
|
97
|
+
#
|
98
|
+
# The `args` will be joined together into a command line.
|
99
|
+
#
|
100
|
+
# It is expected that you will use the matchers defined in
|
101
|
+
# IntegrationMatchers in your integration tests:
|
102
|
+
#
|
103
|
+
# @example Check output of 'ls' includes a string 'foo.txt'
|
104
|
+
# it "lists files" do
|
105
|
+
# command('ls').should have_output('foo.txt')
|
106
|
+
# end
|
107
|
+
#
|
108
|
+
# @example More complicated
|
109
|
+
# context "long format" do
|
110
|
+
# it "lists files with timestamps" do
|
111
|
+
# command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
|
112
|
+
# end
|
113
|
+
# end
|
114
|
+
#
|
115
|
+
# @param [Array<String>] args
|
116
|
+
#
|
117
|
+
# @overload command(*args, options={})
|
118
|
+
# If the last element of `args` is a Hash it will be used for
|
119
|
+
# options.
|
120
|
+
#
|
121
|
+
# The :env option specifies the command line environment to
|
122
|
+
# use for the command. By default this will be the value of
|
123
|
+
# the Ruby process's own `ENV` variable. If running in a
|
124
|
+
# context in which the `integration_env` method is defined,
|
125
|
+
# its return value will be merged on top of `ENV`. An
|
126
|
+
# explicitly provided :env option will again be merged on top.
|
127
|
+
#
|
128
|
+
# The :cwd option specifies the working directory to start in.
|
129
|
+
# It defaults to the value of <tt>Dir.pwd</tt>
|
130
|
+
#
|
131
|
+
# @param [Array<String>] args
|
132
|
+
# @param [Hash] options
|
133
|
+
# @option options [Hash] env the shell environment to spawn the command with
|
134
|
+
# @option options [Hash] cwd the directory to execute the command in
|
135
|
+
def command *args
|
136
|
+
a = args.flatten.compact
|
137
|
+
options = (a.last.is_a?(Hash) ? a.pop : {})
|
138
|
+
|
139
|
+
env = ENV.to_hash.dup
|
140
|
+
env.merge!(integration_env) if respond_to?(:integration_env)
|
141
|
+
env.merge!(options[:env] || {})
|
142
|
+
|
143
|
+
cwd = options[:cwd]
|
144
|
+
cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
|
145
|
+
|
146
|
+
IntegrationTestRunner.new(a, cwd: cwd, env: env)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
@@ -0,0 +1,207 @@
|
|
1
|
+
module Wukong
|
2
|
+
module SpecHelpers
|
3
|
+
|
4
|
+
# A class for running commands and capturing their STDOUT, STDERR,
|
5
|
+
# and exit code. This class is designed to work with the matchers
|
6
|
+
# defined in IntegrationTestMatchers.
|
7
|
+
class IntegrationTestMatcher
|
8
|
+
|
9
|
+
# The runner used to run the actual commands.
|
10
|
+
attr_accessor :runner
|
11
|
+
|
12
|
+
# An array of expectations about the output of the runner.
|
13
|
+
attr_accessor :expectations
|
14
|
+
|
15
|
+
# The expectation which caused failure.
|
16
|
+
attr_accessor :failed_expectation
|
17
|
+
|
18
|
+
# Return whether or not the given command's output matches
|
19
|
+
# expectations.
|
20
|
+
#
|
21
|
+
# If an expectation failes to match, the `failed_expectation`
|
22
|
+
# attribute will be set accordingly.
|
23
|
+
#
|
24
|
+
# @param [IntegrationTestRunner] runner
|
25
|
+
# @return [true, false]
|
26
|
+
def matches?(runner)
|
27
|
+
self.runner = runner
|
28
|
+
runner.run!
|
29
|
+
expectations.each do |expectation|
|
30
|
+
unless output.send(match_function(expectation), expectation)
|
31
|
+
self.failed_expectation = expectation
|
32
|
+
return false
|
33
|
+
end
|
34
|
+
end
|
35
|
+
true
|
36
|
+
end
|
37
|
+
|
38
|
+
# Create a matcher on the given expectations. Each expectation
|
39
|
+
# can be either a String or a Regexp. Strings will be tested
|
40
|
+
# for inclusion in the output, Regexps will be tested for a
|
41
|
+
# match against the output.
|
42
|
+
#
|
43
|
+
# @param [Array<String,Regexp>] expectations
|
44
|
+
def initialize *expectations
|
45
|
+
self.expectations = expectations
|
46
|
+
end
|
47
|
+
|
48
|
+
# :nodoc:
|
49
|
+
def failure_message
|
50
|
+
"From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n #{failed_expectation}#{formatted_error_output}"
|
51
|
+
end
|
52
|
+
|
53
|
+
# :nodoc:
|
54
|
+
def negative_failure_message
|
55
|
+
"Expected #{output_description} of #{runner.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
|
56
|
+
end
|
57
|
+
|
58
|
+
# :nodoc:
|
59
|
+
def formatted_output
|
60
|
+
output.split("\n").map { |line| ' ' + line }.join("\n")
|
61
|
+
end
|
62
|
+
|
63
|
+
# :nodoc:
|
64
|
+
def formatted_error_output
|
65
|
+
output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{runner.stdout}" : "\n\nSTDERR was\n\n#{runner.stderr}"
|
66
|
+
end
|
67
|
+
|
68
|
+
# :nodoc:
|
69
|
+
def formatted_command
|
70
|
+
" $ #{runner.cmd}"
|
71
|
+
end
|
72
|
+
|
73
|
+
# :nodoc:
|
74
|
+
def formatted_env
|
75
|
+
[' {'].tap do |lines|
|
76
|
+
runner.env.each_pair do |key, value|
|
77
|
+
if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
|
78
|
+
lines << " #{key} => #{value},"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
lines << ' }'
|
82
|
+
end.join("\n")
|
83
|
+
end
|
84
|
+
|
85
|
+
# :nodoc:
|
86
|
+
def match_function expectation
|
87
|
+
expectation.is_a?(Regexp) ? :match : :include?
|
88
|
+
end
|
89
|
+
|
90
|
+
# :nodoc:
|
91
|
+
def match_type
|
92
|
+
failed_expectation.is_a?(Regexp) ? 'match' : 'include'
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
# A matcher for the STDOUT of a command.
|
98
|
+
class StdoutMatcher < IntegrationTestMatcher
|
99
|
+
|
100
|
+
# Picks the STDOUT of the command.
|
101
|
+
def output
|
102
|
+
runner.stdout
|
103
|
+
end
|
104
|
+
|
105
|
+
# :nodoc:
|
106
|
+
def output_description
|
107
|
+
"STDOUT"
|
108
|
+
end
|
109
|
+
|
110
|
+
def description
|
111
|
+
"have the correct #{output_description}"
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
# A matcher for the STDOUT of a command.
|
117
|
+
class StderrMatcher < IntegrationTestMatcher
|
118
|
+
|
119
|
+
# Picks the STDOUT of the command.
|
120
|
+
def output
|
121
|
+
runner.stderr
|
122
|
+
end
|
123
|
+
|
124
|
+
# :nodoc:
|
125
|
+
def output_description
|
126
|
+
"STDERR"
|
127
|
+
end
|
128
|
+
|
129
|
+
def description
|
130
|
+
"print an appropriate error message on #{output_description}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
# A matcher for the exit code of a command.
|
135
|
+
class ExitCodeMatcher < IntegrationTestMatcher
|
136
|
+
|
137
|
+
# Initialize this matcher with the given `code`.
|
138
|
+
#
|
139
|
+
# If `code` is the symbol <tt>:non_zero</tt> then the
|
140
|
+
# expectation will be any non-zero exit code.
|
141
|
+
#
|
142
|
+
# @param [Integer,Symbol] code
|
143
|
+
def initialize code
|
144
|
+
if code == :non_zero
|
145
|
+
@expected_code = :non_zero
|
146
|
+
else
|
147
|
+
@expected_code = code.to_i
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
# Return whether or not the given command's exit code matches
|
152
|
+
# the expectation.
|
153
|
+
#
|
154
|
+
# @param [IntegrationTestRunner] runner
|
155
|
+
# @return [true, false]
|
156
|
+
def matches?(runner)
|
157
|
+
self.runner = runner
|
158
|
+
runner.run!
|
159
|
+
if non_zero_exit_code?
|
160
|
+
@failed = true if runner.exit_code == 0
|
161
|
+
else
|
162
|
+
@failed = true if runner.exit_code != expected_exit_code
|
163
|
+
end
|
164
|
+
@failed ? false : true
|
165
|
+
end
|
166
|
+
|
167
|
+
# :nodoc:
|
168
|
+
def failure_message
|
169
|
+
"From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description} Got #{runner.exit_code} instead.#{formatted_error_output}"
|
170
|
+
end
|
171
|
+
|
172
|
+
# :nodoc:
|
173
|
+
def negative_failure_message
|
174
|
+
"From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
|
175
|
+
end
|
176
|
+
|
177
|
+
# :nodoc:
|
178
|
+
def non_zero_exit_code?
|
179
|
+
@expected_code == :non_zero
|
180
|
+
end
|
181
|
+
|
182
|
+
# :nodoc:
|
183
|
+
def expected_exit_code
|
184
|
+
(@expected_code || 0).to_i
|
185
|
+
end
|
186
|
+
|
187
|
+
# :nodoc:
|
188
|
+
def expected_exit_code_description
|
189
|
+
if non_zero_exit_code?
|
190
|
+
"a non-zero exit code"
|
191
|
+
else
|
192
|
+
"an exit code of #{expected_exit_code}"
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
# :nodoc:
|
197
|
+
def description
|
198
|
+
"exit with #{expected_exit_code_description}"
|
199
|
+
end
|
200
|
+
|
201
|
+
# :nodoc:
|
202
|
+
def output_description
|
203
|
+
"STDOUT"
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|