wukong 3.0.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/Gemfile +1 -1
- data/README.md +253 -45
- data/bin/wu +34 -0
- data/bin/wu-source +5 -0
- data/examples/Gemfile +0 -1
- data/examples/deploy_pack/Gemfile +0 -1
- data/examples/improver/tweet_summary.rb +73 -0
- data/examples/ruby_project/Gemfile +0 -1
- data/examples/splitter.rb +94 -0
- data/examples/twitter.rb +5 -0
- data/lib/hanuman.rb +1 -1
- data/lib/hanuman/graph.rb +39 -22
- data/lib/hanuman/stage.rb +46 -13
- data/lib/hanuman/tree.rb +67 -0
- data/lib/wukong.rb +6 -1
- data/lib/wukong/dataflow.rb +19 -48
- data/lib/wukong/driver.rb +176 -65
- data/lib/wukong/{local → driver}/event_machine_driver.rb +1 -13
- data/lib/wukong/driver/wiring.rb +68 -0
- data/lib/wukong/local.rb +6 -4
- data/lib/wukong/local/runner.rb +14 -16
- data/lib/wukong/local/stdio_driver.rb +72 -12
- data/lib/wukong/processor.rb +1 -30
- data/lib/wukong/runner.rb +2 -0
- data/lib/wukong/runner/command_runner.rb +44 -0
- data/lib/wukong/source.rb +33 -0
- data/lib/wukong/source/source_driver.rb +74 -0
- data/lib/wukong/source/source_runner.rb +38 -0
- data/lib/wukong/spec_helpers/shared_examples.rb +0 -1
- data/lib/wukong/spec_helpers/unit_tests.rb +6 -5
- data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +4 -14
- data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +7 -8
- data/lib/wukong/version.rb +1 -1
- data/lib/wukong/widget/echo.rb +55 -0
- data/lib/wukong/widget/{processors.rb → extract.rb} +0 -106
- data/lib/wukong/widget/filters.rb +15 -0
- data/lib/wukong/widget/logger.rb +56 -0
- data/lib/wukong/widget/operators.rb +82 -0
- data/lib/wukong/widget/reducers.rb +2 -0
- data/lib/wukong/widget/reducers/improver.rb +71 -0
- data/lib/wukong/widget/reducers/join_xml.rb +37 -0
- data/lib/wukong/widget/serializers.rb +21 -6
- data/lib/wukong/widgets.rb +6 -3
- data/spec/hanuman/graph_spec.rb +73 -10
- data/spec/hanuman/stage_spec.rb +15 -0
- data/spec/hanuman/tree_spec.rb +119 -0
- data/spec/spec_helper.rb +13 -1
- data/spec/support/example_test_helpers.rb +0 -1
- data/spec/support/model_test_helpers.rb +1 -1
- data/spec/support/shared_context_for_graphs.rb +57 -0
- data/spec/support/shared_examples_for_builders.rb +8 -15
- data/spec/wukong/driver_spec.rb +152 -0
- data/spec/wukong/local/runner_spec.rb +1 -12
- data/spec/wukong/local/stdio_driver_spec.rb +73 -0
- data/spec/wukong/processor_spec.rb +0 -1
- data/spec/wukong/runner_spec.rb +2 -2
- data/spec/wukong/source_spec.rb +6 -0
- data/spec/wukong/widget/extract_spec.rb +101 -0
- data/spec/wukong/widget/logger_spec.rb +23 -0
- data/spec/wukong/widget/operators_spec.rb +25 -0
- data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
- data/spec/wukong/wu-source_spec.rb +32 -0
- data/spec/wukong/wu_spec.rb +14 -0
- data/wukong.gemspec +1 -2
- metadata +45 -28
- data/lib/wukong/local/tcp_driver.rb +0 -47
- data/spec/wu/geo/geolocated_spec.rb +0 -247
- data/spec/wukong/widget/processors_spec.rb +0 -125
|
@@ -1,125 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
describe "Processors" do
|
|
4
|
-
|
|
5
|
-
let(:hsh) { { "hi" => "there", "top" => { "lower" => { "lowest" => "value" } } } }
|
|
6
|
-
let(:ary) { ['1', 2, 'three'] }
|
|
7
|
-
|
|
8
|
-
context :logger do
|
|
9
|
-
it_behaves_like "a processor", :named => :logger
|
|
10
|
-
|
|
11
|
-
it "logs each event at the 'info' level by default" do
|
|
12
|
-
log = mock("logger")
|
|
13
|
-
log.should_receive(:info).with('hi there')
|
|
14
|
-
log.should_receive(:info).with('buddy')
|
|
15
|
-
processor(:logger) do |proc|
|
|
16
|
-
proc.stub!(:log).and_return(log)
|
|
17
|
-
end.given('hi there', 'buddy').should emit(0).records
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
it "logs each event at the a desired level set with an argument" do
|
|
21
|
-
log = mock("logger")
|
|
22
|
-
log.should_receive(:debug).with('hi there')
|
|
23
|
-
log.should_receive(:debug).with('buddy')
|
|
24
|
-
processor(:logger, level: :debug) do |proc|
|
|
25
|
-
proc.stub!(:log).and_return(log)
|
|
26
|
-
end.given('hi there', 'buddy').should emit(0).records
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
context :extract do
|
|
31
|
-
subject { processor(:extract) }
|
|
32
|
-
|
|
33
|
-
it_behaves_like 'a processor', :named => :extract
|
|
34
|
-
|
|
35
|
-
context "on a string" do
|
|
36
|
-
it "emits the string with no arguments" do
|
|
37
|
-
processor(:extract).given('hi there', 'buddy').should emit('hi there', 'buddy')
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
context "on a Fixnum" do
|
|
41
|
-
it "emits the number with no arguments" do
|
|
42
|
-
processor(:extract).given(3, 3.0).should emit(3, 3.0)
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
context "on a Hash" do
|
|
46
|
-
it "emits the hash with no arguments" do
|
|
47
|
-
processor(:extract).given(hsh).should emit(hsh)
|
|
48
|
-
end
|
|
49
|
-
it "can extract a key" do
|
|
50
|
-
processor(:extract, part: 'hi').given(hsh).should emit('there')
|
|
51
|
-
end
|
|
52
|
-
it "emits nil when the value of the key is nil" do
|
|
53
|
-
processor(:extract, part: 'bye').given(hsh).should emit(nil)
|
|
54
|
-
end
|
|
55
|
-
it "can extract a nested key" do
|
|
56
|
-
processor(:extract, part: 'top.lower.lowest').given(hsh).should emit('value')
|
|
57
|
-
end
|
|
58
|
-
it "emits nil when the value of this nested key is nil" do
|
|
59
|
-
processor(:extract, part: 'foo.bar.baz').given(hsh).should emit(nil)
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
context "on an Array" do
|
|
63
|
-
it "emits the array with no arguments" do
|
|
64
|
-
processor(:extract).given(ary).should emit(ary)
|
|
65
|
-
end
|
|
66
|
-
it "can extract the nth value with an integer argument" do
|
|
67
|
-
processor(:extract, part: 2).given(ary).should emit(2)
|
|
68
|
-
end
|
|
69
|
-
it "can extract the nth value with a string argument" do
|
|
70
|
-
processor(:extract, part: '2').given(ary).should emit(2)
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
context "on JSON" do
|
|
74
|
-
let(:garbage) { '{"239823:' }
|
|
75
|
-
it "emits the JSON with no arguments" do
|
|
76
|
-
processor(:extract).given_json(hsh).should emit_json(hsh)
|
|
77
|
-
end
|
|
78
|
-
it "will skip badly formed records" do
|
|
79
|
-
processor(:extract).given(garbage).should emit(garbage)
|
|
80
|
-
end
|
|
81
|
-
it "can extract a key" do
|
|
82
|
-
processor(:extract, part: 'hi').given_json(hsh).should emit('there')
|
|
83
|
-
end
|
|
84
|
-
it "can extract a nested key" do
|
|
85
|
-
processor(:extract, part: 'top.lower.lowest').given_json(hsh).should emit('value')
|
|
86
|
-
end
|
|
87
|
-
it "emits nil when the record is missing the key" do
|
|
88
|
-
processor(:extract, part: 'foo.bar.baz').given_json(hsh).should emit(nil)
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
|
-
context "on delimited data" do
|
|
92
|
-
it "emits the row with no arguments" do
|
|
93
|
-
processor(:extract).given_delimited('|', ary).should emit(ary.map(&:to_s).join('|'))
|
|
94
|
-
end
|
|
95
|
-
it "can extract the nth value with an integer argument" do
|
|
96
|
-
processor(:extract, part: 2, separator: '|').given_delimited('|', ary).should emit('2')
|
|
97
|
-
end
|
|
98
|
-
it "can extract nth value with a string argument" do
|
|
99
|
-
processor(:extract, part: '2', separator: '|').given_delimited('|', ary).should emit('2')
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
|
-
context "on TSV" do
|
|
103
|
-
it "emits the TSV with no arguments" do
|
|
104
|
-
processor(:extract).given_tsv(ary).should emit(ary.map(&:to_s).join("\t"))
|
|
105
|
-
end
|
|
106
|
-
it "can extract the nth value with an integer argument" do
|
|
107
|
-
processor(:extract, part: 2).given_tsv(ary).should emit('2')
|
|
108
|
-
end
|
|
109
|
-
it "can extract the nth value with a string argument" do
|
|
110
|
-
processor(:extract, part: '2').given_tsv(ary).should emit('2')
|
|
111
|
-
end
|
|
112
|
-
end
|
|
113
|
-
context "on CSV" do
|
|
114
|
-
it "emits the CSV with no arguments" do
|
|
115
|
-
processor(:extract).given_csv(ary).should emit(ary.map(&:to_s).join(","))
|
|
116
|
-
end
|
|
117
|
-
it "can extract the nth value with an integer argument" do
|
|
118
|
-
processor(:extract, part: 2, separator: ',').given_csv(ary).should emit('2')
|
|
119
|
-
end
|
|
120
|
-
it "can extract the nth value with a string argument" do
|
|
121
|
-
processor(:extract, part: '2', separator: ',').given_csv(ary).should emit('2')
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
|
-
end
|
|
125
|
-
end
|