redstorm 0.6.6 → 0.7.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +6 -1
  3. data/README.md +8 -7
  4. data/examples/dsl/exclamation_topology.rb +2 -4
  5. data/examples/dsl/exclamation_topology2.rb +4 -5
  6. data/examples/dsl/hello_world_topology.rb +7 -0
  7. data/examples/dsl/kafka_topology.rb +5 -1
  8. data/examples/dsl/redis_word_count_topology.rb +5 -9
  9. data/examples/dsl/ruby_version_topology.rb +2 -0
  10. data/examples/dsl/word_count_topology.rb +4 -5
  11. data/examples/trident/word_count_query.rb +33 -0
  12. data/examples/trident/word_count_topology.rb +153 -0
  13. data/ivy/storm_dependencies.xml +1 -1
  14. data/ivy/topology_dependencies.xml +3 -2
  15. data/lib/red_storm.rb +5 -2
  16. data/lib/red_storm/configurator.rb +12 -0
  17. data/lib/red_storm/dsl/batch_bolt.rb +34 -0
  18. data/lib/red_storm/dsl/batch_committer_bolt.rb +9 -0
  19. data/lib/red_storm/dsl/batch_spout.rb +53 -0
  20. data/lib/red_storm/dsl/bolt.rb +7 -2
  21. data/lib/red_storm/dsl/output_collector.rb +8 -0
  22. data/lib/red_storm/dsl/spout.rb +3 -1
  23. data/lib/red_storm/dsl/topology.rb +2 -2
  24. data/lib/red_storm/dsl/tuple.rb +2 -0
  25. data/lib/red_storm/topology_launcher.rb +14 -10
  26. data/lib/red_storm/version.rb +1 -1
  27. data/redstorm.gemspec +1 -0
  28. data/src/main/redstorm/storm/jruby/JRubyBatchBolt.java +53 -35
  29. data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java +77 -42
  30. data/src/main/redstorm/storm/jruby/JRubyBolt.java +54 -34
  31. data/src/main/redstorm/storm/jruby/JRubySpout.java +62 -40
  32. data/src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java +57 -35
  33. data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java +6 -17
  34. data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java +14 -26
  35. data/src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java +60 -37
  36. data/src/main/redstorm/storm/jruby/JRubyTridentFunction.java +66 -0
  37. metadata +16 -23
  38. data/lib/red_storm/proxy/batch_bolt.rb +0 -63
  39. data/lib/red_storm/proxy/batch_committer_bolt.rb +0 -52
  40. data/lib/red_storm/proxy/batch_spout.rb +0 -59
  41. data/lib/red_storm/proxy/bolt.rb +0 -63
  42. data/lib/red_storm/proxy/proxy_function.rb +0 -40
  43. data/lib/red_storm/proxy/spout.rb +0 -87
  44. data/lib/red_storm/proxy/transactional_committer_spout.rb +0 -47
  45. data/lib/red_storm/proxy/transactional_spout.rb +0 -46
  46. data/src/main/redstorm/storm/jruby/JRubyProxyFunction.java +0 -51
@@ -1,52 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.coordination.BatchOutputCollector'
4
- java_import 'backtype.storm.task.TopologyContext'
5
- java_import 'backtype.storm.coordination.IBatchBolt'
6
- java_import 'backtype.storm.transactional.ICommitter'
7
- java_import 'backtype.storm.topology.OutputFieldsDeclarer'
8
- java_import 'backtype.storm.tuple.Tuple'
9
- java_import 'java.util.Map'
10
-
11
- module Backtype
12
- java_import 'backtype.storm.Config'
13
- end
14
-
15
- java_package 'redstorm.proxy'
16
-
17
- class BatchCommitterBolt
18
- java_implements 'ICommitter, IBatchBolt'
19
-
20
- java_signature 'IBatchCommitterBolt (String base_class_path, String real_bolt_class_name)'
21
- def initialize(base_class_path, real_bolt_class_name)
22
- @real_bolt = Object.module_eval(real_bolt_class_name).new
23
- rescue NameError
24
- require base_class_path
25
- @real_bolt = Object.module_eval(real_bolt_class_name).new
26
- end
27
-
28
- java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
29
- def prepare(conf, context, collector, id)
30
- @real_bolt.prepare(conf, context, collector, id)
31
- end
32
-
33
- java_signature 'void execute(Tuple)'
34
- def execute(tuple)
35
- @real_bolt.execute(tuple)
36
- end
37
-
38
- java_signature 'void finishBatch()'
39
- def finishBatch
40
- @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
41
- end
42
-
43
- java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
44
- def declareOutputFields(declarer)
45
- @real_bolt.declare_output_fields(declarer)
46
- end
47
-
48
- java_signature 'Map<String, Object> getComponentConfiguration()'
49
- def getComponentConfiguration
50
- @real_bolt.get_component_configuration
51
- end
52
- end
@@ -1,59 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.task.TopologyContext'
4
- java_import 'storm.trident.operation.TridentCollector'
5
- java_import 'storm.trident.spout.IBatchSpout'
6
- java_import 'backtype.storm.tuple.Fields'
7
- java_import 'java.util.Map'
8
-
9
- module Backtype
10
- java_import 'backtype.storm.Config'
11
- end
12
-
13
- java_package 'redstorm.proxy'
14
-
15
- # the Spout class is a proxy to the real spout to avoid having to deal with all the
16
- # Java artifacts when creating a spout.
17
-
18
- class BatchSpout
19
- java_implements IBatchSpout
20
-
21
- java_signature 'IBatchSpout (String base_class_path, String real_spout_class_name)'
22
- def initialize(base_class_path, real_spout_class_name)
23
- @real_spout = Object.module_eval(real_spout_class_name).new
24
- rescue NameError
25
- require base_class_path
26
- @real_spout = Object.module_eval(real_spout_class_name).new
27
- end
28
-
29
- java_signature 'void open(Map, TopologyContext)'
30
- def open(conf, context)
31
- @real_spout.open(conf, context) if @real_spout.respond_to?(:open)
32
- end
33
-
34
- java_signature 'void emitBatch(long, TridentCollector)'
35
- def emitBatch(batch_id, collector)
36
- @real_spout.emit_batch(batch_id, collector)
37
- end
38
-
39
- java_signature 'void close()'
40
- def close
41
- @real_spout.close if @real_spout.respond_to?(:close)
42
- end
43
-
44
- java_signature 'void ack(long)'
45
- def ack(batch_id)
46
- @real_spout.ack(batch_id) if @real_spout.respond_to?(:ack)
47
- end
48
-
49
- java_signature 'Fields getOutputFields()'
50
- def getOutputFields()
51
- @real_spout.get_output_fields
52
- end
53
-
54
- java_signature 'Map<String, Object> getComponentConfiguration()'
55
- def getComponentConfiguration
56
- @real_spout.get_component_configuration
57
- end
58
-
59
- end
@@ -1,63 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.task.OutputCollector'
4
- java_import 'backtype.storm.task.TopologyContext'
5
- java_import 'backtype.storm.topology.IRichBolt'
6
- java_import 'backtype.storm.topology.OutputFieldsDeclarer'
7
- java_import 'backtype.storm.tuple.Tuple'
8
- java_import 'backtype.storm.tuple.Fields'
9
- java_import 'backtype.storm.tuple.Values'
10
- java_import 'java.util.Map'
11
- module Backtype
12
- java_import 'backtype.storm.Config'
13
- end
14
-
15
- java_package 'redstorm.proxy'
16
-
17
- # the Bolt class is a proxy to the real bolt to avoid having to deal with all the
18
- # Java artifacts when creating a bolt.
19
- #
20
- # The real bolt class implementation must define these methods:
21
- # - prepare(conf, context, collector)
22
- # - execute(tuple)
23
- # - declare_output_fields
24
- #
25
- # and optionnaly:
26
- # - cleanup
27
- #
28
- class Bolt
29
- java_implements IRichBolt
30
-
31
- java_signature 'IRichBolt (String base_class_path, String real_bolt_class_name)'
32
- def initialize(base_class_path, real_bolt_class_name)
33
- @real_bolt = Object.module_eval(real_bolt_class_name).new
34
- rescue NameError
35
- require base_class_path
36
- @real_bolt = Object.module_eval(real_bolt_class_name).new
37
- end
38
-
39
- java_signature 'void prepare(Map, TopologyContext, OutputCollector)'
40
- def prepare(conf, context, collector)
41
- @real_bolt.prepare(conf, context, collector)
42
- end
43
-
44
- java_signature 'void execute(Tuple)'
45
- def execute(tuple)
46
- @real_bolt.execute(tuple)
47
- end
48
-
49
- java_signature 'void cleanup()'
50
- def cleanup
51
- @real_bolt.cleanup if @real_bolt.respond_to?(:cleanup)
52
- end
53
-
54
- java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
55
- def declareOutputFields(declarer)
56
- @real_bolt.declare_output_fields(declarer)
57
- end
58
-
59
- java_signature 'Map<String, Object> getComponentConfiguration()'
60
- def getComponentConfiguration
61
- @real_bolt.get_component_configuration
62
- end
63
- end
@@ -1,40 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'storm.trident.tuple.TridentTuple'
4
- java_import 'storm.trident.operation.TridentCollector'
5
- java_import 'storm.trident.operation.TridentOperationContext'
6
- java_import 'storm.trident.operation.Function'
7
- java_import 'java.util.Map'
8
-
9
- module Backtype
10
- java_import 'backtype.storm.Config'
11
- end
12
-
13
- java_package 'redstorm.proxy'
14
-
15
- class ProxyFunction
16
- java_implements Function
17
-
18
- java_signature 'Function (String base_class_path, String real_class_name)'
19
- def initialize(base_class_path, real_class_name)
20
- @real = Object.module_eval(real_class_name).new
21
- rescue NameError
22
- require base_class_path
23
- @real = Object.module_eval(real_class_name).new
24
- end
25
-
26
- java_signature 'void execute(TridentTuple, TridentCollector)'
27
- def execute(_trident_tuple, _trident_collector)
28
- @real.execute(_trident_tuple, _trident_collector)
29
- end
30
-
31
- java_signature 'void cleanup()'
32
- def cleanup()
33
- @real.cleanup()
34
- end
35
-
36
- java_signature 'void prepare(Map, TridentOperationContext)'
37
- def prepare(_map, _trident_operation_context)
38
- @real.prepare(_map, _trident_operation_context)
39
- end
40
- end
@@ -1,87 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.spout.SpoutOutputCollector'
4
- java_import 'backtype.storm.task.TopologyContext'
5
- java_import 'backtype.storm.topology.IRichSpout'
6
- java_import 'backtype.storm.topology.OutputFieldsDeclarer'
7
- java_import 'backtype.storm.tuple.Tuple'
8
- java_import 'backtype.storm.tuple.Fields'
9
- java_import 'backtype.storm.tuple.Values'
10
- java_import 'java.util.Map'
11
- module Backtype
12
- java_import 'backtype.storm.Config'
13
- end
14
-
15
- java_package 'redstorm.proxy'
16
-
17
- # the Spout class is a proxy to the real spout to avoid having to deal with all the
18
- # Java artifacts when creating a spout.
19
- #
20
- # The real spout class implementation must define these methods:
21
- # - open(conf, context, collector)
22
- # - next_tuple
23
- # - declare_output_fields
24
- #
25
- # and optionnaly:
26
- # - ack(msg_id)
27
- # - fail(msg_id)
28
- # - close
29
- #
30
-
31
- class Spout
32
- java_implements IRichSpout
33
-
34
- java_signature 'IRichSpout (String base_class_path, String real_spout_class_name)'
35
- def initialize(base_class_path, real_spout_class_name)
36
- @real_spout = Object.module_eval(real_spout_class_name).new
37
- rescue NameError
38
- require base_class_path
39
- @real_spout = Object.module_eval(real_spout_class_name).new
40
- end
41
-
42
- java_signature 'void open(Map, TopologyContext, SpoutOutputCollector)'
43
- def open(conf, context, collector)
44
- @real_spout.open(conf, context, collector)
45
- end
46
-
47
- java_signature 'void close()'
48
- def close
49
- @real_spout.close if @real_spout.respond_to?(:close)
50
- end
51
-
52
- java_signature 'void activate()'
53
- def activate
54
- @real_spout.activate if @real_spout.respond_to?(:activate)
55
- end
56
-
57
- java_signature 'void deactivate()'
58
- def deactivate
59
- @real_spout.deactivate if @real_spout.respond_to?(:deactivate)
60
- end
61
-
62
- java_signature 'void nextTuple()'
63
- def nextTuple
64
- @real_spout.next_tuple
65
- end
66
-
67
- java_signature 'void ack(Object)'
68
- def ack(msg_id)
69
- @real_spout.ack(msg_id) if @real_spout.respond_to?(:ack)
70
- end
71
-
72
- java_signature 'void fail(Object)'
73
- def fail(msg_id)
74
- @real_spout.fail(msg_id) if @real_spout.respond_to?(:fail)
75
- end
76
-
77
- java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
78
- def declareOutputFields(declarer)
79
- @real_spout.declare_output_fields(declarer)
80
- end
81
-
82
- java_signature 'Map<String, Object> getComponentConfiguration()'
83
- def getComponentConfiguration
84
- @real_spout.get_component_configuration
85
- end
86
-
87
- end
@@ -1,47 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.task.TopologyContext'
4
- java_import 'backtype.storm.transactional.ITransactionalSpout'
5
- java_import 'backtype.storm.transactional.ICommitterTransactionalSpout'
6
- java_import 'backtype.storm.topology.OutputFieldsDeclarer'
7
- java_import 'java.util.Map'
8
-
9
- module Backtype
10
- java_import 'backtype.storm.Config'
11
- end
12
-
13
- java_package 'redstorm.proxy'
14
-
15
-
16
- class TransactionalCommitterSpout
17
- java_implements 'ICommitterTransactionalSpout'
18
-
19
- java_signature 'ICommitterTransactionalSpout (String base_class_path, String real_spout_class_name)'
20
- def initialize(base_class_path, real_spout_class_name)
21
- @real_spout = Object.module_eval(real_spout_class_name).new
22
- rescue NameError
23
- require base_class_path
24
- @real_spout = Object.module_eval(real_spout_class_name).new
25
- end
26
-
27
- java_signature 'ICommitterTransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
28
- def getEmitter(conf, context)
29
- @real_spout.get_emitter(conf, context)
30
- end
31
-
32
- java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
33
- def getCoordinator(conf, context)
34
- @real_spout.get_coordinator(conf, context)
35
- end
36
-
37
- java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
38
- def declareOutputFields(declarer)
39
- @real_spout.declare_output_fields(declarer)
40
- end
41
-
42
- java_signature 'Map<String, Object> getComponentConfiguration()'
43
- def getComponentConfiguration
44
- @real_spout.get_component_configuration
45
- end
46
-
47
- end
@@ -1,46 +0,0 @@
1
- require 'java'
2
-
3
- java_import 'backtype.storm.task.TopologyContext'
4
- java_import 'backtype.storm.transactional.ITransactionalSpout'
5
- java_import 'backtype.storm.topology.OutputFieldsDeclarer'
6
- java_import 'java.util.Map'
7
-
8
- module Backtype
9
- java_import 'backtype.storm.Config'
10
- end
11
-
12
- java_package 'redstorm.proxy'
13
-
14
-
15
- class TransactionalSpout
16
- java_implements 'ITransactionalSpout'
17
-
18
- java_signature 'ITransactionalSpout (String base_class_path, String real_spout_class_name)'
19
- def initialize(base_class_path, real_spout_class_name)
20
- @real_spout = Object.module_eval(real_spout_class_name).new
21
- rescue NameError
22
- require base_class_path
23
- @real_spout = Object.module_eval(real_spout_class_name).new
24
- end
25
-
26
- java_signature 'ITransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
27
- def getEmitter(conf, context)
28
- @real_spout.get_emitter(conf, context)
29
- end
30
-
31
- java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
32
- def getCoordinator(conf, context)
33
- @real_spout.get_coordinator(conf, context)
34
- end
35
-
36
- java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
37
- def declareOutputFields(declarer)
38
- @real_spout.declare_output_fields(declarer)
39
- end
40
-
41
- java_signature 'Map<String, Object> getComponentConfiguration()'
42
- def getComponentConfiguration
43
- @real_spout.get_component_configuration
44
- end
45
-
46
- end
@@ -1,51 +0,0 @@
1
- package redstorm.storm.jruby;
2
-
3
- import storm.trident.tuple.TridentTuple;
4
- import storm.trident.operation.TridentCollector;
5
- import java.util.Map;
6
- import storm.trident.operation.TridentOperationContext;
7
- import storm.trident.operation.Function;
8
-
9
- public class JRubyProxyFunction implements Function {
10
- Function _proxy;
11
- String _realClassName;
12
- String _baseClassPath;
13
-
14
- public JRubyProxyFunction(final String baseClassPath, final String realClassName) {
15
- _baseClassPath = baseClassPath;
16
- _realClassName = realClassName;
17
- }
18
-
19
-
20
- @Override
21
- public void execute(final TridentTuple _tridentTuple, final TridentCollector _tridentCollector) {
22
- if(_proxy == null) {
23
- _proxy = newProxy(_baseClassPath, _realClassName);
24
- }
25
- _proxy.execute(_tridentTuple, _tridentCollector);
26
- }
27
-
28
- @Override
29
- public void cleanup() {
30
- _proxy.cleanup();
31
- }
32
-
33
- @Override
34
- public void prepare(final Map _map, final TridentOperationContext _tridentOperationContext) {
35
- if(_proxy == null) {
36
- _proxy = newProxy(_baseClassPath, _realClassName);
37
- }
38
- _proxy.prepare(_map, _tridentOperationContext);
39
- }
40
-
41
-
42
- private static Function newProxy(final String baseClassPath, final String realClassName) {
43
- try {
44
- redstorm.proxy.ProxyFunction proxy = new redstorm.proxy.ProxyFunction(baseClassPath, realClassName);
45
- return proxy;
46
- }
47
- catch (Exception e) {
48
- throw new RuntimeException(e);
49
- }
50
- }
51
- }