kb-redstorm 0.6.5 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/CHANGELOG.md +9 -0
  2. data/README.md +206 -103
  3. data/examples/native/cluster_word_count_topology.rb +5 -5
  4. data/examples/native/local_exclamation_topology.rb +8 -8
  5. data/examples/native/local_exclamation_topology2.rb +7 -7
  6. data/examples/native/local_redis_word_count_topology.rb +7 -8
  7. data/examples/native/local_word_count_topology.rb +5 -5
  8. data/examples/simple/exclamation_topology.rb +7 -11
  9. data/examples/simple/exclamation_topology2.rb +10 -12
  10. data/examples/simple/hello_world_topology.rb +22 -0
  11. data/examples/simple/kafka_topology.rb +15 -15
  12. data/examples/simple/redis_word_count_topology.rb +3 -5
  13. data/examples/simple/ruby_version_topology.rb +7 -1
  14. data/examples/simple/word_count_topology.rb +8 -10
  15. data/ivy/settings.xml +1 -0
  16. data/ivy/storm_dependencies.xml +8 -0
  17. data/ivy/topology_dependencies.xml +7 -0
  18. data/lib/red_storm/application.rb +7 -5
  19. data/lib/red_storm/configurator.rb +1 -1
  20. data/lib/red_storm/proxy/batch_bolt.rb +63 -0
  21. data/lib/red_storm/proxy/batch_committer_bolt.rb +52 -0
  22. data/lib/red_storm/proxy/batch_spout.rb +12 -24
  23. data/lib/red_storm/proxy/proxy_function.rb +1 -9
  24. data/lib/red_storm/proxy/transactional_committer_spout.rb +47 -0
  25. data/lib/red_storm/proxy/transactional_spout.rb +46 -0
  26. data/lib/red_storm/simple_drpc_topology.rb +2 -2
  27. data/lib/red_storm/simple_topology.rb +14 -4
  28. data/lib/red_storm/topology_launcher.rb +16 -0
  29. data/lib/red_storm/version.rb +1 -1
  30. data/lib/tasks/red_storm.rake +69 -108
  31. data/redstorm.gemspec +24 -0
  32. data/src/main/redstorm/storm/jruby/JRubyBatchBolt.java +90 -0
  33. data/src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java +9 -0
  34. data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java +25 -26
  35. data/src/main/redstorm/storm/jruby/JRubyProxyFunction.java +1 -9
  36. data/src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java +90 -0
  37. data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java +31 -0
  38. data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java +44 -0
  39. data/src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java +89 -0
  40. metadata +80 -62
  41. data/examples/native/Gemfile +0 -2
@@ -0,0 +1,90 @@
1
+ package redstorm.storm.jruby;
2
+
3
+ import backtype.storm.task.OutputCollector;
4
+ import backtype.storm.task.TopologyContext;
5
+ import backtype.storm.transactional.TransactionAttempt;
6
+ import backtype.storm.topology.base.BaseTransactionalBolt;
7
+ import backtype.storm.coordination.BatchOutputCollector;
8
+ import backtype.storm.coordination.IBatchBolt;
9
+ import backtype.storm.topology.OutputFieldsDeclarer;
10
+ import backtype.storm.tuple.Tuple;
11
+ import backtype.storm.tuple.Fields;
12
+ import java.util.Map;
13
+
14
+ /**
15
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
16
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
17
+ * the bolts to the workers. JRuby does not yet support serialization from Java
18
+ * (Java serialization call on a JRuby class).
19
+ *
20
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after
21
+ * deserialization at the worker and in the declareOutputFields method which is called once before
22
+ * serialization at topology creation.
23
+ */
24
+ public class JRubyTransactionalBolt extends BaseTransactionalBolt {
25
+ IBatchBolt _proxyBolt;
26
+ String _realBoltClassName;
27
+ String _baseClassPath;
28
+ String[] _fields;
29
+
30
+ /**
31
+ * create a new JRubyBolt
32
+ *
33
+ * @param baseClassPath the topology/project base JRuby class file path
34
+ * @param realBoltClassName the fully qualified JRuby bolt implementation class name
35
+ */
36
+ public JRubyTransactionalBolt(String baseClassPath, String realBoltClassName, String[] fields) {
37
+ _baseClassPath = baseClassPath;
38
+ _realBoltClassName = realBoltClassName;
39
+ _fields = fields;
40
+ }
41
+
42
+ @Override
43
+ public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final TransactionAttempt id) {
44
+ // create instance of the jruby class here, after deserialization in the workers.
45
+ _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
46
+ _proxyBolt.prepare(stormConf, context, collector, id);
47
+ }
48
+
49
+ @Override
50
+ public void execute(Tuple input) {
51
+ _proxyBolt.execute(input);
52
+ }
53
+
54
+ @Override
55
+ public void finishBatch() {
56
+ _proxyBolt.finishBatch();
57
+ }
58
+
59
+ @Override
60
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
61
+ // declareOutputFields is executed in the topology creation time, before serialisation.
62
+ // do not set the _proxyBolt instance variable here to avoid JRuby serialization
63
+ // issues. Just create tmp bolt instance to call declareOutputFields.
64
+ if (_fields.length > 0) {
65
+ declarer.declare(new Fields(_fields));
66
+ } else {
67
+ IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
68
+ bolt.declareOutputFields(declarer);
69
+ }
70
+ }
71
+
72
+ @Override
73
+ public Map<String, Object> getComponentConfiguration() {
74
+ // getComponentConfiguration is executed in the topology creation time, before serialisation.
75
+ // do not set the _proxyBolt instance variable here to avoid JRuby serialization
76
+ // issues. Just create tmp bolt instance to call declareOutputFields.
77
+ IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
78
+ return bolt.getComponentConfiguration();
79
+ }
80
+
81
+ private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
82
+ try {
83
+ redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
84
+ return proxy;
85
+ }
86
+ catch (Exception e) {
87
+ throw new RuntimeException(e);
88
+ }
89
+ }
90
+ }
@@ -0,0 +1,31 @@
1
+ package redstorm.storm.jruby;
2
+
3
+ import backtype.storm.coordination.IBatchBolt;
4
+ import backtype.storm.transactional.TransactionAttempt;
5
+ import backtype.storm.transactional.ICommitter;
6
+
7
+ /**
8
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
9
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
10
+ * the bolts to the workers. JRuby does not yet support serialization from Java
11
+ * (Java serialization call on a JRuby class).
12
+ *
13
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after
14
+ * deserialization at the worker and in the declareOutputFields method which is called once before
15
+ * serialization at topology creation.
16
+ */
17
+ public class JRubyTransactionalCommitterBolt extends JRubyTransactionalBolt implements ICommitter {
18
+ public JRubyTransactionalCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
19
+ super(baseClassPath, realBoltClassName, fields);
20
+ }
21
+
22
+ private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
23
+ try {
24
+ redstorm.proxy.BatchCommitterBolt proxy = new redstorm.proxy.BatchCommitterBolt(baseClassPath, realBoltClassName);
25
+ return proxy;
26
+ }
27
+ catch (Exception e) {
28
+ throw new RuntimeException(e);
29
+ }
30
+ }
31
+ }
@@ -0,0 +1,44 @@
1
+ package redstorm.storm.jruby;
2
+
3
+ import backtype.storm.transactional.ICommitterTransactionalSpout;
4
+ import backtype.storm.transactional.ITransactionalSpout;
5
+ import backtype.storm.task.TopologyContext;
6
+ import java.util.Map;
7
+
8
+ /**
9
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
10
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
11
+ * the bolts to the workers. JRuby does not yet support serialization from Java
12
+ * (Java serialization call on a JRuby class).
13
+ *
14
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after
15
+ * deserialization at the worker and in the declareOutputFields method which is called once before
16
+ * serialization at topology creation.
17
+ */
18
+ public class JRubyTransactionalCommitterSpout extends JRubyTransactionalSpout implements ICommitterTransactionalSpout {
19
+
20
+ ICommitterTransactionalSpout _proxySpout;
21
+
22
+ public JRubyTransactionalCommitterSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
23
+ super(baseClassPath, realSpoutClassName, fields);
24
+ }
25
+
26
+ @Override
27
+ public ICommitterTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
28
+ // create instance of the jruby class here, after deserialization in the workers.
29
+ if (_proxySpout == null) {
30
+ _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
31
+ }
32
+ return _proxySpout.getEmitter(conf, context);
33
+ }
34
+
35
+ private static ICommitterTransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
36
+ try {
37
+ redstorm.proxy.TransactionalCommitterSpout proxy = new redstorm.proxy.TransactionalCommitterSpout(baseClassPath, realSpoutClassName);
38
+ return proxy;
39
+ }
40
+ catch (Exception e) {
41
+ throw new RuntimeException(e);
42
+ }
43
+ }
44
+ }
@@ -0,0 +1,89 @@
1
+ package redstorm.storm.jruby;
2
+
3
+ import backtype.storm.spout.SpoutOutputCollector;
4
+ import backtype.storm.task.TopologyContext;
5
+ import backtype.storm.topology.base.BaseTransactionalSpout;
6
+ import backtype.storm.topology.OutputFieldsDeclarer;
7
+ import backtype.storm.transactional.ITransactionalSpout;
8
+ import backtype.storm.tuple.Tuple;
9
+ import backtype.storm.tuple.Fields;
10
+ import java.util.Map;
11
+
12
+ /**
13
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
14
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
15
+ * the spout to the workers. JRuby does not yet support serialization from Java
16
+ * (Java serialization call on a JRuby class).
17
+ *
18
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after
19
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which
20
+ * are called once before serialization at topology creation.
21
+ */
22
+ public class JRubyTransactionalSpout extends BaseTransactionalSpout {
23
+ ITransactionalSpout _proxySpout;
24
+ String _realSpoutClassName;
25
+ String _baseClassPath;
26
+ String[] _fields;
27
+
28
+ /**
29
+ * create a new JRubySpout
30
+ *
31
+ * @param baseClassPath the topology/project base JRuby class file path
32
+ * @param realSpoutClassName the fully qualified JRuby spout implementation class name
33
+ */
34
+ public JRubyTransactionalSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
35
+ _baseClassPath = baseClassPath;
36
+ _realSpoutClassName = realSpoutClassName;
37
+ _fields = fields;
38
+ }
39
+
40
+ @Override
41
+ public ITransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) {
42
+ // create instance of the jruby class here, after deserialization in the workers.
43
+ if (_proxySpout == null) {
44
+ _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
45
+ }
46
+ return _proxySpout.getCoordinator(conf, context);
47
+ }
48
+
49
+ @Override
50
+ public ITransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
51
+ // create instance of the jruby class here, after deserialization in the workers.
52
+ if (_proxySpout == null) {
53
+ _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
54
+ }
55
+ return _proxySpout.getEmitter(conf, context);
56
+ }
57
+
58
+ @Override
59
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
60
+ // declareOutputFields is executed in the topology creation time before serialisation.
61
+ // do not set the _proxySpout instance variable here to avoid JRuby serialization
62
+ // issues. Just create tmp spout instance to call declareOutputFields.
63
+ if (_fields.length > 0) {
64
+ declarer.declare(new Fields(_fields));
65
+ } else {
66
+ ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
67
+ spout.declareOutputFields(declarer);
68
+ }
69
+ }
70
+
71
+ @Override
72
+ public Map<String, Object> getComponentConfiguration() {
73
+ // getComponentConfiguration is executed in the topology creation time before serialisation.
74
+ // do not set the _proxySpout instance variable here to avoid JRuby serialization
75
+ // issues. Just create tmp spout instance to call declareOutputFields.
76
+ ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
77
+ return spout.getComponentConfiguration();
78
+ }
79
+
80
+ private static ITransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
81
+ try {
82
+ redstorm.proxy.TransactionalSpout proxy = new redstorm.proxy.TransactionalSpout(baseClassPath, realSpoutClassName);
83
+ return proxy;
84
+ }
85
+ catch (Exception e) {
86
+ throw new RuntimeException(e);
87
+ }
88
+ }
89
+ }
metadata CHANGED
@@ -1,79 +1,84 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: kb-redstorm
3
- version: !ruby/object:Gem::Version
4
- prerelease:
5
- version: 0.6.5
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 6
8
+ - 6
9
+ version: 0.6.6
6
10
  platform: ruby
7
- authors:
11
+ authors:
8
12
  - Colin Surprenant
9
- autorequire:
13
+ autorequire:
10
14
  bindir: bin
11
15
  cert_chain: []
12
- date: 2013-04-29 00:00:00.000000000 Z
13
- dependencies:
14
- - !ruby/object:Gem::Dependency
16
+
17
+ date: 2013-06-11 00:00:00 -04:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
15
21
  name: rspec
16
- version_requirements: !ruby/object:Gem::Requirement
17
- requirements:
18
- - - "~>"
19
- - !ruby/object:Gem::Version
20
- version: 2.11.0
21
- none: false
22
- requirement: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - "~>"
25
- - !ruby/object:Gem::Version
26
- version: 2.11.0
27
- none: false
28
22
  prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ~>
26
+ - !ruby/object:Gem::Version
27
+ segments:
28
+ - 2
29
+ - 11
30
+ - 0
31
+ version: 2.11.0
29
32
  type: :development
30
- - !ruby/object:Gem::Dependency
33
+ version_requirements: *id001
34
+ - !ruby/object:Gem::Dependency
31
35
  name: rake
32
- version_requirements: !ruby/object:Gem::Requirement
33
- requirements:
34
- - - ">="
35
- - !ruby/object:Gem::Version
36
- version: !binary |-
37
- MA==
38
- none: false
39
- requirement: !ruby/object:Gem::Requirement
40
- requirements:
41
- - - ">="
42
- - !ruby/object:Gem::Version
43
- version: !binary |-
44
- MA==
45
- none: false
46
36
  prerelease: false
37
+ requirement: &id002 !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ segments:
42
+ - 0
43
+ version: "0"
47
44
  type: :runtime
45
+ version_requirements: *id002
48
46
  description: JRuby integration & DSL for the Storm distributed realtime computation system
49
- email:
47
+ email:
50
48
  - colin.surprenant@gmail.com
51
- executables:
49
+ executables:
52
50
  - redstorm
53
51
  extensions: []
52
+
54
53
  extra_rdoc_files: []
55
- files:
56
- - lib/red_storm.rb
54
+
55
+ files:
57
56
  - lib/red_storm/application.rb
58
57
  - lib/red_storm/configuration.rb
59
58
  - lib/red_storm/configurator.rb
60
59
  - lib/red_storm/environment.rb
61
60
  - lib/red_storm/loggable.rb
61
+ - lib/red_storm/proxy/batch_bolt.rb
62
+ - lib/red_storm/proxy/batch_committer_bolt.rb
63
+ - lib/red_storm/proxy/batch_spout.rb
64
+ - lib/red_storm/proxy/bolt.rb
65
+ - lib/red_storm/proxy/proxy_function.rb
66
+ - lib/red_storm/proxy/spout.rb
67
+ - lib/red_storm/proxy/transactional_committer_spout.rb
68
+ - lib/red_storm/proxy/transactional_spout.rb
62
69
  - lib/red_storm/simple_bolt.rb
63
70
  - lib/red_storm/simple_drpc_topology.rb
64
71
  - lib/red_storm/simple_spout.rb
65
72
  - lib/red_storm/simple_topology.rb
66
73
  - lib/red_storm/topology_launcher.rb
67
74
  - lib/red_storm/version.rb
68
- - lib/red_storm/proxy/batch_spout.rb
69
- - lib/red_storm/proxy/bolt.rb
70
- - lib/red_storm/proxy/proxy_function.rb
71
- - lib/red_storm/proxy/spout.rb
75
+ - lib/red_storm.rb
72
76
  - lib/tasks/red_storm.rake
73
77
  - ivy/settings.xml
78
+ - ivy/storm_dependencies.xml
79
+ - ivy/topology_dependencies.xml
74
80
  - examples/native/cluster_word_count_topology.rb
75
81
  - examples/native/exclamation_bolt.rb
76
- - examples/native/Gemfile
77
82
  - examples/native/local_exclamation_topology.rb
78
83
  - examples/native/local_exclamation_topology2.rb
79
84
  - examples/native/local_redis_word_count_topology.rb
@@ -81,12 +86,13 @@ files:
81
86
  - examples/native/random_sentence_spout.rb
82
87
  - examples/native/split_sentence_bolt.rb
83
88
  - examples/native/word_count_bolt.rb
84
- - examples/shell/shell_topology.rb
85
89
  - examples/shell/resources/splitsentence.py
86
90
  - examples/shell/resources/storm.py
91
+ - examples/shell/shell_topology.rb
87
92
  - examples/simple/exclamation_bolt.rb
88
93
  - examples/simple/exclamation_topology.rb
89
94
  - examples/simple/exclamation_topology2.rb
95
+ - examples/simple/hello_world_topology.rb
90
96
  - examples/simple/kafka_topology.rb
91
97
  - examples/simple/random_sentence_spout.rb
92
98
  - examples/simple/redis_word_count_topology.rb
@@ -94,41 +100,53 @@ files:
94
100
  - examples/simple/split_sentence_bolt.rb
95
101
  - examples/simple/word_count_bolt.rb
96
102
  - examples/simple/word_count_topology.rb
103
+ - src/main/redstorm/storm/jruby/JRubyBatchBolt.java
104
+ - src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java
97
105
  - src/main/redstorm/storm/jruby/JRubyBatchSpout.java
98
106
  - src/main/redstorm/storm/jruby/JRubyBolt.java
99
107
  - src/main/redstorm/storm/jruby/JRubyProxyFunction.java
100
108
  - src/main/redstorm/storm/jruby/JRubyShellBolt.java
101
109
  - src/main/redstorm/storm/jruby/JRubyShellSpout.java
102
110
  - src/main/redstorm/storm/jruby/JRubySpout.java
111
+ - src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java
112
+ - src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java
113
+ - src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java
114
+ - src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java
103
115
  - bin/redstorm
116
+ - redstorm.gemspec
104
117
  - Rakefile
105
118
  - README.md
106
119
  - CHANGELOG.md
107
120
  - LICENSE.md
121
+ has_rdoc: true
108
122
  homepage: https://github.com/colinsurprenant/redstorm
109
123
  licenses: []
110
- post_install_message:
124
+
125
+ post_install_message:
111
126
  rdoc_options: []
112
- require_paths:
127
+
128
+ require_paths:
113
129
  - lib
114
- required_ruby_version: !ruby/object:Gem::Requirement
115
- requirements:
130
+ required_ruby_version: !ruby/object:Gem::Requirement
131
+ requirements:
116
132
  - - ">="
117
- - !ruby/object:Gem::Version
118
- version: !binary |-
119
- MA==
120
- none: false
121
- required_rubygems_version: !ruby/object:Gem::Requirement
122
- requirements:
133
+ - !ruby/object:Gem::Version
134
+ segments:
135
+ - 0
136
+ version: "0"
137
+ required_rubygems_version: !ruby/object:Gem::Requirement
138
+ requirements:
123
139
  - - ">="
124
- - !ruby/object:Gem::Version
125
- version: !binary |-
126
- MA==
127
- none: false
140
+ - !ruby/object:Gem::Version
141
+ segments:
142
+ - 0
143
+ version: "0"
128
144
  requirements: []
145
+
129
146
  rubyforge_project: redstorm
130
- rubygems_version: 1.8.24
131
- signing_key:
147
+ rubygems_version: 1.3.6
148
+ signing_key:
132
149
  specification_version: 3
133
150
  summary: JRuby on Storm
134
151
  test_files: []
152
+
@@ -1,2 +0,0 @@
1
- source :rubygems
2
- gem 'redis'