redstorm 0.6.4 → 0.6.5

data/src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java

@@ -0,0 +1,9 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.transactional.ICommitter;
+
+public class JRubyBatchCommitterBolt extends JRubyBatchBolt implements ICommitter {
+  public JRubyBatchCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    super(baseClassPath, realBoltClassName, fields);
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java

@@ -0,0 +1,88 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.tuple.Fields;
+import backtype.storm.task.TopologyContext;
+import storm.trident.operation.TridentCollector;
+import storm.trident.spout.IBatchSpout;
+import java.util.Map;
+
+/**
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the spout to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
+ * are called once before serialization at topology creation. 
+ */
+public class JRubyBatchSpout implements IBatchSpout {
+  IBatchSpout _proxySpout;
+  String _realSpoutClassName;
+  String _baseClassPath;
+  
+  /**
+   * create a new JRubySpout
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   */
+  public JRubyBatchSpout(String baseClassPath, String realSpoutClassName) {
+    _baseClassPath = baseClassPath;
+    _realSpoutClassName = realSpoutClassName;
+  }
+
+  @Override
+  public void open(final Map conf, final TopologyContext context) {
+    // create instance of the jruby proxy class here, after deserialization in the workers.
+    _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    _proxySpout.open(conf, context);
+  }
+
+  @Override
+  public void emitBatch(long batchId, TridentCollector collector) {
+    _proxySpout.emitBatch(batchId, collector);
+  } 
+
+  @Override
+  public void close() {
+    _proxySpout.close();
+  }
+
+  @Override
+  public void ack(long batchId) {
+    _proxySpout.ack(batchId);
+  }
+
+  @Override
+  public Fields getOutputFields() {
+    if (_proxySpout == null) {
+      // getOutputFields is executed in the topology creation time before serialisation.
+      // do not set the _proxySpout instance variable here to avoid JRuby serialization
+      // issues. Just create tmp spout instance to call getOutputFields.
+      IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      return spout.getOutputFields();
+    } else {
+      return _proxySpout.getOutputFields();
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+ 
+  private static IBatchSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.BatchSpout proxy = new redstorm.proxy.BatchSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyProxyFunction.java

@@ -0,0 +1,51 @@
+package redstorm.storm.jruby;
+
+import storm.trident.tuple.TridentTuple;
+import storm.trident.operation.TridentCollector;
+import java.util.Map;
+import storm.trident.operation.TridentOperationContext;
+import storm.trident.operation.Function;
+
+public class JRubyProxyFunction implements Function {
+  Function _proxy;
+  String _realClassName;
+  String _baseClassPath;
+
+  public JRubyProxyFunction(final String baseClassPath, final String realClassName) {
+    _baseClassPath = baseClassPath;
+    _realClassName = realClassName;
+  }
+
+
+  @Override
+  public void execute(final TridentTuple _tridentTuple, final TridentCollector _tridentCollector) {
+    if(_proxy == null) {
+      _proxy = newProxy(_baseClassPath, _realClassName);
+    }
+    _proxy.execute(_tridentTuple, _tridentCollector);
+  }
+
+  @Override
+  public void cleanup() {
+    _proxy.cleanup();
+  }
+
+  @Override
+  public void prepare(final Map _map, final TridentOperationContext _tridentOperationContext) {
+    if(_proxy == null) {
+      _proxy = newProxy(_baseClassPath, _realClassName);
+    }
+    _proxy.prepare(_map, _tridentOperationContext);
+  }
+
+
+  private static Function newProxy(final String baseClassPath, final String realClassName) {
+    try {
+      redstorm.proxy.ProxyFunction proxy = new redstorm.proxy.ProxyFunction(baseClassPath, realClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyShellSpout.java

@@ -1,4 +1,4 @@
-package backtype.storm.clojure;
+package redstorm.storm.jruby;
 
 import backtype.storm.spout.ShellSpout;
 import backtype.storm.topology.IRichSpout;

data/src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java

@@ -0,0 +1,90 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.transactional.TransactionAttempt;
+import backtype.storm.topology.base.BaseTransactionalBolt;
+import backtype.storm.coordination.BatchOutputCollector;
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalBolt extends BaseTransactionalBolt {
+  IBatchBolt _proxyBolt;
+  String _realBoltClassName;
+  String _baseClassPath; 
+  String[] _fields;
+
+ /**
+   * create a new JRubyBolt
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   */
+  public JRubyTransactionalBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realBoltClassName = realBoltClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final TransactionAttempt id) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    _proxyBolt.prepare(stormConf, context, collector, id);
+  }
+
+  @Override
+  public void execute(Tuple input) {
+    _proxyBolt.execute(input);
+  }
+
+  @Override
+  public void finishBatch() {
+    _proxyBolt.finishBatch();
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java

@@ -0,0 +1,31 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.transactional.TransactionAttempt;
+import backtype.storm.transactional.ICommitter;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalCommitterBolt extends JRubyTransactionalBolt implements ICommitter {
+  public JRubyTransactionalCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    super(baseClassPath, realBoltClassName, fields);
+  }
+
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchCommitterBolt proxy = new redstorm.proxy.BatchCommitterBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java

@@ -0,0 +1,44 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.transactional.ICommitterTransactionalSpout;
+import backtype.storm.transactional.ITransactionalSpout;
+import backtype.storm.task.TopologyContext;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalCommitterSpout extends JRubyTransactionalSpout implements ICommitterTransactionalSpout {
+
+  ICommitterTransactionalSpout _proxySpout;
+  
+  public JRubyTransactionalCommitterSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    super(baseClassPath, realSpoutClassName, fields);
+  }
+
+  @Override
+  public ICommitterTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getEmitter(conf, context);
+  }
+
+  private static ICommitterTransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.TransactionalCommitterSpout proxy = new redstorm.proxy.TransactionalCommitterSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java

@@ -0,0 +1,89 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseTransactionalSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.transactional.ITransactionalSpout;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the spout to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
+ * are called once before serialization at topology creation. 
+ */
+public class JRubyTransactionalSpout extends BaseTransactionalSpout {
+  ITransactionalSpout _proxySpout;
+  String _realSpoutClassName;
+  String _baseClassPath;
+  String[] _fields;
+  
+  /**
+   * create a new JRubySpout
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   */
+  public JRubyTransactionalSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public ITransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getCoordinator(conf, context);
+  }
+
+  @Override
+  public ITransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+  	if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getEmitter(conf, context);
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      spout.declareOutputFields(declarer);
+    }
+  }  
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+ 
+  private static ITransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.TransactionalSpout proxy = new redstorm.proxy.TransactionalSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: redstorm
 version: !ruby/object:Gem::Version
+  version: 0.6.5
   prerelease:
-  version: 0.6.4
 platform: ruby
 authors:
 - Colin Surprenant
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-10-19 00:00:00.000000000 Z
+date: 2013-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.11.0
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.11.0
     none: false
@@ -31,15 +31,17 @@ dependencies:
   name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: !binary |-
+          MA==
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: !binary |-
+          MA==
     none: false
   prerelease: false
   type: :runtime
@@ -58,17 +60,25 @@ files:
 - lib/red_storm/environment.rb
 - lib/red_storm/loggable.rb
 - lib/red_storm/simple_bolt.rb
+- lib/red_storm/simple_drpc_topology.rb
 - lib/red_storm/simple_spout.rb
 - lib/red_storm/simple_topology.rb
 - lib/red_storm/topology_launcher.rb
 - lib/red_storm/version.rb
+- lib/red_storm/proxy/batch_bolt.rb
+- lib/red_storm/proxy/batch_committer_bolt.rb
+- lib/red_storm/proxy/batch_spout.rb
 - lib/red_storm/proxy/bolt.rb
+- lib/red_storm/proxy/proxy_function.rb
 - lib/red_storm/proxy/spout.rb
+- lib/red_storm/proxy/transactional_committer_spout.rb
+- lib/red_storm/proxy/transactional_spout.rb
 - lib/tasks/red_storm.rake
 - ivy/settings.xml
+- ivy/storm_dependencies.xml
+- ivy/topology_dependencies.xml
 - examples/native/cluster_word_count_topology.rb
 - examples/native/exclamation_bolt.rb
-- examples/native/Gemfile
 - examples/native/local_exclamation_topology.rb
 - examples/native/local_exclamation_topology2.rb
 - examples/native/local_redis_word_count_topology.rb
@@ -82,6 +92,7 @@ files:
 - examples/simple/exclamation_bolt.rb
 - examples/simple/exclamation_topology.rb
 - examples/simple/exclamation_topology2.rb
+- examples/simple/hello_world_topology.rb
 - examples/simple/kafka_topology.rb
 - examples/simple/random_sentence_spout.rb
 - examples/simple/redis_word_count_topology.rb
@@ -89,11 +100,20 @@ files:
 - examples/simple/split_sentence_bolt.rb
 - examples/simple/word_count_bolt.rb
 - examples/simple/word_count_topology.rb
+- src/main/redstorm/storm/jruby/JRubyBatchBolt.java
+- src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java
+- src/main/redstorm/storm/jruby/JRubyBatchSpout.java
 - src/main/redstorm/storm/jruby/JRubyBolt.java
+- src/main/redstorm/storm/jruby/JRubyProxyFunction.java
 - src/main/redstorm/storm/jruby/JRubyShellBolt.java
 - src/main/redstorm/storm/jruby/JRubyShellSpout.java
 - src/main/redstorm/storm/jruby/JRubySpout.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java
 - bin/redstorm
+- redstorm.gemspec
 - Rakefile
 - README.md
 - CHANGELOG.md
@@ -106,15 +126,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: !binary |-
+        MA==
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: !binary |-
+        MA==
   none: false
 requirements: []
 rubyforge_project: redstorm
@@ -123,4 +145,3 @@ signing_key:
 specification_version: 3
 summary: JRuby on Storm
 test_files: []
-...