wukong 3.0.1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. data/.gitignore +1 -0
  2. data/Gemfile +1 -1
  3. data/README.md +253 -45
  4. data/bin/wu +34 -0
  5. data/bin/wu-source +5 -0
  6. data/examples/Gemfile +0 -1
  7. data/examples/deploy_pack/Gemfile +0 -1
  8. data/examples/improver/tweet_summary.rb +73 -0
  9. data/examples/ruby_project/Gemfile +0 -1
  10. data/examples/splitter.rb +94 -0
  11. data/examples/twitter.rb +5 -0
  12. data/lib/hanuman.rb +1 -1
  13. data/lib/hanuman/graph.rb +39 -22
  14. data/lib/hanuman/stage.rb +46 -13
  15. data/lib/hanuman/tree.rb +67 -0
  16. data/lib/wukong.rb +6 -1
  17. data/lib/wukong/dataflow.rb +19 -48
  18. data/lib/wukong/driver.rb +176 -65
  19. data/lib/wukong/{local → driver}/event_machine_driver.rb +1 -13
  20. data/lib/wukong/driver/wiring.rb +68 -0
  21. data/lib/wukong/local.rb +6 -4
  22. data/lib/wukong/local/runner.rb +14 -16
  23. data/lib/wukong/local/stdio_driver.rb +72 -12
  24. data/lib/wukong/processor.rb +1 -30
  25. data/lib/wukong/runner.rb +2 -0
  26. data/lib/wukong/runner/command_runner.rb +44 -0
  27. data/lib/wukong/source.rb +33 -0
  28. data/lib/wukong/source/source_driver.rb +74 -0
  29. data/lib/wukong/source/source_runner.rb +38 -0
  30. data/lib/wukong/spec_helpers/shared_examples.rb +0 -1
  31. data/lib/wukong/spec_helpers/unit_tests.rb +6 -5
  32. data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb +4 -14
  33. data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb +7 -8
  34. data/lib/wukong/version.rb +1 -1
  35. data/lib/wukong/widget/echo.rb +55 -0
  36. data/lib/wukong/widget/{processors.rb → extract.rb} +0 -106
  37. data/lib/wukong/widget/filters.rb +15 -0
  38. data/lib/wukong/widget/logger.rb +56 -0
  39. data/lib/wukong/widget/operators.rb +82 -0
  40. data/lib/wukong/widget/reducers.rb +2 -0
  41. data/lib/wukong/widget/reducers/improver.rb +71 -0
  42. data/lib/wukong/widget/reducers/join_xml.rb +37 -0
  43. data/lib/wukong/widget/serializers.rb +21 -6
  44. data/lib/wukong/widgets.rb +6 -3
  45. data/spec/hanuman/graph_spec.rb +73 -10
  46. data/spec/hanuman/stage_spec.rb +15 -0
  47. data/spec/hanuman/tree_spec.rb +119 -0
  48. data/spec/spec_helper.rb +13 -1
  49. data/spec/support/example_test_helpers.rb +0 -1
  50. data/spec/support/model_test_helpers.rb +1 -1
  51. data/spec/support/shared_context_for_graphs.rb +57 -0
  52. data/spec/support/shared_examples_for_builders.rb +8 -15
  53. data/spec/wukong/driver_spec.rb +152 -0
  54. data/spec/wukong/local/runner_spec.rb +1 -12
  55. data/spec/wukong/local/stdio_driver_spec.rb +73 -0
  56. data/spec/wukong/processor_spec.rb +0 -1
  57. data/spec/wukong/runner_spec.rb +2 -2
  58. data/spec/wukong/source_spec.rb +6 -0
  59. data/spec/wukong/widget/extract_spec.rb +101 -0
  60. data/spec/wukong/widget/logger_spec.rb +23 -0
  61. data/spec/wukong/widget/operators_spec.rb +25 -0
  62. data/spec/wukong/widget/reducers/join_xml_spec.rb +25 -0
  63. data/spec/wukong/wu-source_spec.rb +32 -0
  64. data/spec/wukong/wu_spec.rb +14 -0
  65. data/wukong.gemspec +1 -2
  66. metadata +45 -28
  67. data/lib/wukong/local/tcp_driver.rb +0 -47
  68. data/spec/wu/geo/geolocated_spec.rb +0 -247
  69. data/spec/wukong/widget/processors_spec.rb +0 -125
data/spec/wukong/widget/processors_spec.rb DELETED
@@ -1,125 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe "Processors" do
4
-
5
- let(:hsh) { { "hi" => "there", "top" => { "lower" => { "lowest" => "value" } } } }
6
- let(:ary) { ['1', 2, 'three'] }
7
-
8
- context :logger do
9
- it_behaves_like "a processor", :named => :logger
10
-
11
- it "logs each event at the 'info' level by default" do
12
- log = mock("logger")
13
- log.should_receive(:info).with('hi there')
14
- log.should_receive(:info).with('buddy')
15
- processor(:logger) do |proc|
16
- proc.stub!(:log).and_return(log)
17
- end.given('hi there', 'buddy').should emit(0).records
18
- end
19
-
20
- it "logs each event at the a desired level set with an argument" do
21
- log = mock("logger")
22
- log.should_receive(:debug).with('hi there')
23
- log.should_receive(:debug).with('buddy')
24
- processor(:logger, level: :debug) do |proc|
25
- proc.stub!(:log).and_return(log)
26
- end.given('hi there', 'buddy').should emit(0).records
27
- end
28
- end
29
-
30
- context :extract do
31
- subject { processor(:extract) }
32
-
33
- it_behaves_like 'a processor', :named => :extract
34
-
35
- context "on a string" do
36
- it "emits the string with no arguments" do
37
- processor(:extract).given('hi there', 'buddy').should emit('hi there', 'buddy')
38
- end
39
- end
40
- context "on a Fixnum" do
41
- it "emits the number with no arguments" do
42
- processor(:extract).given(3, 3.0).should emit(3, 3.0)
43
- end
44
- end
45
- context "on a Hash" do
46
- it "emits the hash with no arguments" do
47
- processor(:extract).given(hsh).should emit(hsh)
48
- end
49
- it "can extract a key" do
50
- processor(:extract, part: 'hi').given(hsh).should emit('there')
51
- end
52
- it "emits nil when the value of the key is nil" do
53
- processor(:extract, part: 'bye').given(hsh).should emit(nil)
54
- end
55
- it "can extract a nested key" do
56
- processor(:extract, part: 'top.lower.lowest').given(hsh).should emit('value')
57
- end
58
- it "emits nil when the value of this nested key is nil" do
59
- processor(:extract, part: 'foo.bar.baz').given(hsh).should emit(nil)
60
- end
61
- end
62
- context "on an Array" do
63
- it "emits the array with no arguments" do
64
- processor(:extract).given(ary).should emit(ary)
65
- end
66
- it "can extract the nth value with an integer argument" do
67
- processor(:extract, part: 2).given(ary).should emit(2)
68
- end
69
- it "can extract the nth value with a string argument" do
70
- processor(:extract, part: '2').given(ary).should emit(2)
71
- end
72
- end
73
- context "on JSON" do
74
- let(:garbage) { '{"239823:' }
75
- it "emits the JSON with no arguments" do
76
- processor(:extract).given_json(hsh).should emit_json(hsh)
77
- end
78
- it "will skip badly formed records" do
79
- processor(:extract).given(garbage).should emit(garbage)
80
- end
81
- it "can extract a key" do
82
- processor(:extract, part: 'hi').given_json(hsh).should emit('there')
83
- end
84
- it "can extract a nested key" do
85
- processor(:extract, part: 'top.lower.lowest').given_json(hsh).should emit('value')
86
- end
87
- it "emits nil when the record is missing the key" do
88
- processor(:extract, part: 'foo.bar.baz').given_json(hsh).should emit(nil)
89
- end
90
- end
91
- context "on delimited data" do
92
- it "emits the row with no arguments" do
93
- processor(:extract).given_delimited('|', ary).should emit(ary.map(&:to_s).join('|'))
94
- end
95
- it "can extract the nth value with an integer argument" do
96
- processor(:extract, part: 2, separator: '|').given_delimited('|', ary).should emit('2')
97
- end
98
- it "can extract nth value with a string argument" do
99
- processor(:extract, part: '2', separator: '|').given_delimited('|', ary).should emit('2')
100
- end
101
- end
102
- context "on TSV" do
103
- it "emits the TSV with no arguments" do
104
- processor(:extract).given_tsv(ary).should emit(ary.map(&:to_s).join("\t"))
105
- end
106
- it "can extract the nth value with an integer argument" do
107
- processor(:extract, part: 2).given_tsv(ary).should emit('2')
108
- end
109
- it "can extract the nth value with a string argument" do
110
- processor(:extract, part: '2').given_tsv(ary).should emit('2')
111
- end
112
- end
113
- context "on CSV" do
114
- it "emits the CSV with no arguments" do
115
- processor(:extract).given_csv(ary).should emit(ary.map(&:to_s).join(","))
116
- end
117
- it "can extract the nth value with an integer argument" do
118
- processor(:extract, part: 2, separator: ',').given_csv(ary).should emit('2')
119
- end
120
- it "can extract the nth value with a string argument" do
121
- processor(:extract, part: '2', separator: ',').given_csv(ary).should emit('2')
122
- end
123
- end
124
- end
125
- end