redstorm 0.6.6 → 0.7.0.beta1

data/src/main/redstorm/storm/jruby/JRubyBolt.java

@@ -8,81 +8,101 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Fields;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
- * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the bolts to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubyBolt class is a proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
- * deserialization at the worker and in the declareOutputFields method which is called once before 
- * serialization at topology creation. 
+ * Note that the JRuby bolt class is instanciated in the prepare method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubyBolt implements IRichBolt {
-  IRichBolt _proxyBolt;
-  String _realBoltClassName;
-  String _baseClassPath;
-  String[] _fields;
+  private final String _realBoltClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  private transient IRubyObject _ruby_bolt;
+  private transient Ruby __ruby__;
 
   /**
    * create a new JRubyBolt
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   * @param fields the output fields names
    */
   public JRubyBolt(String baseClassPath, String realBoltClassName, String[] fields) {
-    _baseClassPath = baseClassPath;
     _realBoltClassName = realBoltClassName;
     _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
-  public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
-    // create instance of the jruby class here, after deserialization in the workers.
-    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    _proxyBolt.prepare(stormConf, context, collector);
+  public void prepare(final Map conf, final TopologyContext context, final OutputCollector collector) {
+    _ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "prepare", ruby_conf, ruby_context, ruby_collector);
   }
 
   @Override
   public void execute(Tuple input) {
-    _proxyBolt.execute(input);
+    IRubyObject ruby_input = JavaUtil.convertJavaToRuby(__ruby__, input);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "execute", ruby_input);
   }
 
   @Override
   public void cleanup() {
-    _proxyBolt.cleanup();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "cleanup");
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
     // declareOutputFields is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
+    // just create tmp bolt instance to call declareOutputFields.
+
     if (_fields.length > 0) {
       declarer.declare(new Fields(_fields));
     } else {
-      IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-      bolt.declareOutputFields(declarer);
+      IRubyObject ruby_bolt = initialize_ruby_bolt();
+      IRubyObject ruby_declarer = JavaUtil.convertJavaToRuby(__ruby__, declarer);
+      Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "declare_output_fields", ruby_declarer);
     }
   }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
     // getComponentConfiguration is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
-    IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    return bolt.getComponentConfiguration();
+    // just create tmp bolt instance to call getComponentConfiguration.
+
+    IRubyObject ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
 
-  private static IRichBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+  private IRubyObject initialize_ruby_bolt() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.Bolt proxy = new redstorm.proxy.Bolt(baseClassPath, realBoltClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }

data/src/main/redstorm/storm/jruby/JRubySpout.java

@@ -8,100 +8,122 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Fields;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
- * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the spout to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubySpout class is a proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
- * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
- * are called once before serialization at topology creation. 
+ * Note that the JRuby spout class is instanciated in the open method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubySpout implements IRichSpout {
-  IRichSpout _proxySpout;
-  String _realSpoutClassName;
-  String _baseClassPath;
-  String[] _fields;
+  private final String _realSpoutClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  private transient IRubyObject _ruby_spout;
+  private transient Ruby __ruby__;
 
   /**
    * create a new JRubySpout
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   * @param fields the output fields names
    */
   public JRubySpout(String baseClassPath, String realSpoutClassName, String[] fields) {
-    _baseClassPath = baseClassPath;
     _realSpoutClassName = realSpoutClassName;
     _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
   public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
-    // create instance of the jruby proxy class here, after deserialization in the workers.
-    _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    _proxySpout.open(conf, context, collector);
+    _ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "open", ruby_conf, ruby_context, ruby_collector);
   }
 
   @Override
   public void close() {
-    _proxySpout.close();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "close");
   }
 
   @Override
   public void activate() {
-    _proxySpout.activate();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "activate");
   }
 
   @Override
   public void deactivate() {
-    _proxySpout.deactivate();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "deactivate");
   }
 
   @Override
   public void nextTuple() {
-    _proxySpout.nextTuple();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "next_tuple");
   }
 
   @Override
   public void ack(Object msgId) {
-    _proxySpout.ack(msgId);
+    IRubyObject ruby_msg_id = JavaUtil.convertJavaToRuby(__ruby__, msgId);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "ack", ruby_msg_id);
   }
 
   @Override
   public void fail(Object msgId) {
-    _proxySpout.fail(msgId);
+    IRubyObject ruby_msg_id = JavaUtil.convertJavaToRuby(__ruby__, msgId);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "fail", ruby_msg_id);
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
-    // declareOutputFields is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // just create tmp spout instance to call declareOutputFields.
+
     if (_fields.length > 0) {
       declarer.declare(new Fields(_fields));
     } else {
-      IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-      spout.declareOutputFields(declarer);
+      IRubyObject ruby_spout = initialize_ruby_spout();
+      IRubyObject ruby_declarer = JavaUtil.convertJavaToRuby(__ruby__, declarer);
+      Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "declare_output_fields", ruby_declarer);
     }
-  }  
+  }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
-    // getComponentConfiguration is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
-    IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    return spout.getComponentConfiguration();
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // just create tmp spout instance to call getComponentConfiguration.
+
+    IRubyObject ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
-  private static IRichSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+
+  private IRubyObject initialize_ruby_spout() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.Spout proxy = new redstorm.proxy.Spout(baseClassPath, realSpoutClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }

data/src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java

@@ -11,80 +11,102 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Fields;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
- * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the bolts to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubyBolt class is a proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
- * deserialization at the worker and in the declareOutputFields method which is called once before 
- * serialization at topology creation. 
+ * Note that the JRuby bolt class is instanciated in the prepare method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubyTransactionalBolt extends BaseTransactionalBolt {
-  IBatchBolt _proxyBolt;
-  String _realBoltClassName;
-  String _baseClassPath; 
-  String[] _fields;
+  private final String _realBoltClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  protected transient IRubyObject _ruby_bolt;
+  protected transient Ruby __ruby__;
 
- /**
+  /**
    * create a new JRubyBolt
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   * @param fields the output fields names
    */
   public JRubyTransactionalBolt(String baseClassPath, String realBoltClassName, String[] fields) {
-    _baseClassPath = baseClassPath;
     _realBoltClassName = realBoltClassName;
     _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
-  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final TransactionAttempt id) {
-    // create instance of the jruby class here, after deserialization in the workers.
-    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    _proxyBolt.prepare(stormConf, context, collector, id);
+  public void prepare(final Map conf, final TopologyContext context, final BatchOutputCollector collector, final TransactionAttempt id) {
+    _ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    IRubyObject ruby_id = JavaUtil.convertJavaToRuby(__ruby__, id);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "prepare", ruby_conf, ruby_context, ruby_collector, ruby_id);
   }
 
   @Override
   public void execute(Tuple input) {
-    _proxyBolt.execute(input);
+    IRubyObject ruby_input = JavaUtil.convertJavaToRuby(__ruby__, input);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "execute", ruby_input);
   }
 
   @Override
   public void finishBatch() {
-    _proxyBolt.finishBatch();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "finish_batch");
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
     // declareOutputFields is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
+    // just create tmp bolt instance to call declareOutputFields.
+
     if (_fields.length > 0) {
       declarer.declare(new Fields(_fields));
     } else {
-      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-      bolt.declareOutputFields(declarer);
+      IRubyObject ruby_bolt = initialize_ruby_bolt();
+      IRubyObject ruby_declarer = JavaUtil.convertJavaToRuby(__ruby__, declarer);
+      Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "declare_output_fields", ruby_declarer);
     }
   }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
     // getComponentConfiguration is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
-    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    return bolt.getComponentConfiguration();
+    // just create tmp bolt instance to call getComponentConfiguration.
+
+    IRubyObject ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
-  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+
+  protected IRubyObject initialize_ruby_bolt() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java

@@ -5,27 +5,16 @@ import backtype.storm.transactional.TransactionAttempt;
 import backtype.storm.transactional.ICommitter;
 
 /**
- * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the bolts to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubyBolt class is a proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
- * deserialization at the worker and in the declareOutputFields method which is called once before 
- * serialization at topology creation. 
+ * Note that the JRuby bolt class is instanciated in the prepare method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubyTransactionalCommitterBolt extends JRubyTransactionalBolt implements ICommitter {
   public JRubyTransactionalCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
     super(baseClassPath, realBoltClassName, fields);
   }
-
-  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
-    try {
-      redstorm.proxy.BatchCommitterBolt proxy = new redstorm.proxy.BatchCommitterBolt(baseClassPath, realBoltClassName);
-      return proxy;
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
 }