redstorm 0.6.6 → 0.7.0.beta1

data/lib/red_storm/dsl/batch_committer_bolt.rb

@@ -0,0 +1,9 @@
+module RedStorm
+  module DSL
+
+    class BatchCommitterBolt < BatchBolt
+
+      def self.java_proxy; "Java::RedstormStormJruby::JRubyBatchCommitterBolt"; end
+    end
+  end
+end

data/lib/red_storm/dsl/batch_spout.rb

@@ -0,0 +1,53 @@
+module RedStorm
+  module DSL
+
+    class BatchSpout < Spout
+
+      def self.java_proxy; "Java::RedstormStormJruby::JRubyBatchSpout"; end
+
+      def get_output_fields
+        Fields.new(self.class.fields)
+      end
+
+      def self.on_emit_batch(*args, &on_emit_batch_block)
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        method_name = args.first
+
+        self.on_emit_batch_options.merge!(options)
+
+        # indirecting through a lambda defers the method lookup at invocation time
+        # and the performance penalty is negligible
+        body = block_given? ? on_emit_batch_block : lambda{|batch_id, collector| self.send((method_name || :on_emit_batch).to_sym)}
+        define_method(:on_emit_batch, body)
+      end
+
+      # Spout proxy interface
+
+      #
+      # note that in batch spout, ack is for the batch id and not the message id as in the base spout.
+      # TODO maybe rename msg_id to just id in the base spout
+      #
+
+      def emit_batch(batch_id, collector)
+        # TODO this is a TridentCollector, emit should just work by setting @collector
+        # report_error need to be hooked?
+        @collector = collector
+        on_emit_batch(batch_id, collector)
+      end
+
+      def open(config, context)
+        @context = context
+        @config = config
+
+        on_init
+      end
+
+      private
+
+      def self.on_emit_batch_options
+        @on_emit_batch_options ||= {}
+      end
+
+    end
+  end
+end

data/lib/red_storm/dsl/bolt.rb

@@ -3,6 +3,9 @@ require 'red_storm/configurator'
 require 'red_storm/environment'
 require 'pathname'
 
+java_import 'backtype.storm.tuple.Fields'
+java_import 'backtype.storm.tuple.Values'
+
 module RedStorm
   module DSL
 
@@ -11,6 +14,8 @@ module RedStorm
     class Bolt
       attr_reader :collector, :context, :config
 
+      def self.java_proxy; "Java::RedstormStormJruby::JRubyBolt"; end
+
       # DSL class methods
 
       def self.log
@@ -54,11 +59,11 @@ module RedStorm
       end
 
       def unanchored_emit(*values)
-        @collector.emit(Values.new(*values))
+        @collector.emit_tuple(Values.new(*values))
       end
 
       def anchored_emit(tuple, *values)
-        @collector.emit(tuple, Values.new(*values))
+        @collector.emit_anchor_tuple(tuple, Values.new(*values))
       end
 
       def ack(tuple)

data/lib/red_storm/dsl/output_collector.rb

@@ -0,0 +1,8 @@
+require 'java'
+java_import 'backtype.storm.task.OutputCollector'
+
+# make alias methods to specific signatures to avoid selection overhead for heavily overloaded method
+class OutputCollector
+  java_alias :emit_tuple, :emit, [java.lang.Class.for_name("java.util.List")]
+  java_alias :emit_anchor_tuple, :emit, [java.lang.Class.for_name("backtype.storm.tuple.Tuple"), java.lang.Class.for_name("java.util.List")]
+end

data/lib/red_storm/dsl/spout.rb

@@ -11,6 +11,8 @@ module RedStorm
     class Spout
       attr_reader :config, :context, :collector
 
+      def self.java_proxy; "Java::RedstormStormJruby::JRubySpout"; end
+
       # DSL class methods
 
       def self.configure(&configure_block)
@@ -99,7 +101,7 @@ module RedStorm
               unreliable_emit(*values)
             end
           else
-            sleep(0.1)
+            sleep(0.1)  # see https://twitter.com/colinsurprenant/status/406445541904494592
           end
         end
       end

data/lib/red_storm/dsl/topology.rb

@@ -46,7 +46,7 @@ module RedStorm
           elsif is_java?
             @clazz.new(*constructor_args)
           else
-            JRubySpout.new(@clazz.base_class_path, @clazz.name, @output_fields)
+            Object.module_eval(@clazz.java_proxy).new(@clazz.base_class_path, @clazz.name, @output_fields)
           end
         end
       end
@@ -95,7 +95,7 @@ module RedStorm
           elsif is_java?
             @clazz.new(*constructor_args)
           else
-            JRubyBolt.new(@clazz.base_class_path, @clazz.name, @output_fields)
+            Object.module_eval(@clazz.java_proxy).new(@clazz.base_class_path, @clazz.name, @output_fields)
           end
         end
       end

data/lib/red_storm/dsl/tuple.rb

@@ -23,6 +23,8 @@ class TupleImpl
   end
   alias_method :[], :value
 
+  alias_method :values, :getValues # wire directly to avoid method missing indirection
+
   def field_index(field)
     fieldIndex(field.to_s)
   end

data/lib/red_storm/topology_launcher.rb

@@ -2,8 +2,11 @@ require 'java'
 
 # This hack get rif of the "Use RbConfig instead of obsolete and deprecated Config"
 # deprecation warning that is triggered by "java_import 'backtype.storm.Config'".
-Object.send :remove_const, :Config
-Config = RbConfig
+begin
+  Object.send :remove_const, :Config
+  Config = RbConfig
+rescue NameError
+end
 
 # see https://github.com/colinsurprenant/redstorm/issues/7
 module Backtype
@@ -20,14 +23,15 @@ java_import 'backtype.storm.tuple.Fields'
 java_import 'backtype.storm.tuple.Tuple'
 java_import 'backtype.storm.tuple.Values'
 
-java_import 'redstorm.storm.jruby.JRubyBolt'
-java_import 'redstorm.storm.jruby.JRubySpout'
-java_import 'redstorm.storm.jruby.JRubyBatchBolt'
-java_import 'redstorm.storm.jruby.JRubyBatchCommitterBolt'
-java_import 'redstorm.storm.jruby.JRubyBatchSpout'
-java_import 'redstorm.storm.jruby.JRubyTransactionalSpout'
-java_import 'redstorm.storm.jruby.JRubyTransactionalBolt'
-java_import 'redstorm.storm.jruby.JRubyTransactionalCommitterBolt'
+# java_import 'redstorm.storm.jruby.JRubyBolt'
+# java_import 'redstorm.storm.jruby.JRubySpout'
+# java_import 'redstorm.storm.jruby.JRubyBatchBolt'
+# java_import 'redstorm.storm.jruby.JRubyBatchCommitterBolt'
+# java_import 'redstorm.storm.jruby.JRubyBatchSpout'
+# java_import 'redstorm.storm.jruby.JRubyTransactionalSpout'
+# java_import 'redstorm.storm.jruby.JRubyTransactionalBolt'
+# java_import 'redstorm.storm.jruby.JRubyTransactionalCommitterBolt'
+# java_import 'redstorm.storm.jruby.JRubyProxyFunction'
 
 java_package 'redstorm'
 

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.6.6'
+  VERSION = '0.7.0.beta1'
 end

data/redstorm.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/colinsurprenant/redstorm'
   s.summary     = 'JRuby on Storm'
   s.description = 'JRuby integration & DSL for the Storm distributed realtime computation system'
+  s.license     = "Apache 2.0"
 
   s.rubyforge_project = 'redstorm'
 

data/src/main/redstorm/storm/jruby/JRubyBatchBolt.java

@@ -10,81 +10,99 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Fields;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
- * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the bolts to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubyBolt class is a proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
- * deserialization at the worker and in the declareOutputFields method which is called once before 
- * serialization at topology creation. 
+ * Note that the JRuby bolt class is instanciated in the prepare method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubyBatchBolt extends BaseBatchBolt {
-  IBatchBolt _proxyBolt;
-  String _realBoltClassName;
-  String _baseClassPath; 
-  String[] _fields;
+  private final String _realBoltClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  private transient IRubyObject _ruby_bolt;
+  private transient Ruby __ruby__;
 
   /**
    * create a new JRubyBolt
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   * @param fields the output fields names
    */
   public JRubyBatchBolt(String baseClassPath, String realBoltClassName, String[] fields) {
-    _baseClassPath = baseClassPath;
     _realBoltClassName = realBoltClassName;
     _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
-  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final Object id) {
-    // create instance of the jruby class here, after deserialization in the workers.
-    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    _proxyBolt.prepare(stormConf, context, collector, id);
+  public void prepare(final Map conf, final TopologyContext context, final BatchOutputCollector collector, final Object id) {
+    _ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    IRubyObject ruby_id = JavaUtil.convertJavaToRuby(__ruby__, id);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "prepare", ruby_conf, ruby_context, ruby_collector, ruby_id);
   }
 
   @Override
   public void execute(Tuple input) {
-    _proxyBolt.execute(input);
+    IRubyObject ruby_input = JavaUtil.convertJavaToRuby(__ruby__, input);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "execute", ruby_input);
   }
 
   @Override
   public void finishBatch() {
-    _proxyBolt.finishBatch();
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_bolt, "finish_batch");
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
-    // declareOutputFields is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
     if (_fields.length > 0) {
       declarer.declare(new Fields(_fields));
     } else {
-      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-      bolt.declareOutputFields(declarer);
+      IRubyObject ruby_bolt = initialize_ruby_bolt();
+      IRubyObject ruby_declarer = JavaUtil.convertJavaToRuby(__ruby__, declarer);
+      Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "declare_output_fields", ruby_declarer);
     }
   }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
     // getComponentConfiguration is executed in the topology creation time, before serialisation.
-    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
-    // issues. Just create tmp bolt instance to call declareOutputFields.
-    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    return bolt.getComponentConfiguration();
+    // just create tmp bolt instance to call getComponentConfiguration.
+
+    IRubyObject ruby_bolt = initialize_ruby_bolt();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_bolt, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
 
-  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+  private IRubyObject initialize_ruby_bolt() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realBoltClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }

data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java

@@ -6,83 +6,118 @@ import storm.trident.operation.TridentCollector;
 import storm.trident.spout.IBatchSpout;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
- * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the spout to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * the JRubySpout class is a proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization issues of JRuby objects.
+ * JRuby classes do not support Java serialization.
  *
- * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
- * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
- * are called once before serialization at topology creation. 
+ * Note that the JRuby spout class is instanciated in the open method which is called after
+ * deserialization at the worker and in the declareOutputFields & getComponentConfiguration
+ * methods which are called once before serialization at topology creation.
  */
 public class JRubyBatchSpout implements IBatchSpout {
-  IBatchSpout _proxySpout;
-  String _realSpoutClassName;
-  String _baseClassPath;
-  
+  private final String _realSpoutClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  private transient IRubyObject _ruby_spout;
+  private transient Ruby __ruby__;
+
   /**
-   * create a new JRubySpout
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   * create a new JRubyBatchSpout
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   * @param fields the output fields names
    */
   public JRubyBatchSpout(String baseClassPath, String realSpoutClassName) {
-    _baseClassPath = baseClassPath;
     _realSpoutClassName = realSpoutClassName;
+    _fields = null;
+    _bootstrap = "require '" + baseClassPath + "'";
+  }
+
+  /* constructor for compatibility with JRubySpout signature */
+  public JRubyBatchSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
   public void open(final Map conf, final TopologyContext context) {
-    // create instance of the jruby proxy class here, after deserialization in the workers.
-    _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    _proxySpout.open(conf, context);
+    // // create instance of the jruby proxy class here, after deserialization in the workers.
+    // _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    // _proxySpout.open(conf, context);
+
+
+    _ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "open", ruby_conf, ruby_context);
   }
 
   @Override
   public void emitBatch(long batchId, TridentCollector collector) {
-    _proxySpout.emitBatch(batchId, collector);
-  } 
+    // _proxySpout.emitBatch(batchId, collector);
+
+    IRubyObject ruby_batch_id = JavaUtil.convertJavaToRuby(__ruby__, batchId);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "emit_batch", ruby_batch_id, ruby_collector);
+  }
 
   @Override
   public void close() {
-    _proxySpout.close();
+    // _proxySpout.close();
+
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "close");
   }
 
   @Override
   public void ack(long batchId) {
-    _proxySpout.ack(batchId);
+    // _proxySpout.ack(batchId);
+
+    IRubyObject ruby_batch_id = JavaUtil.convertJavaToRuby(__ruby__, batchId);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "ack", ruby_batch_id);
   }
 
   @Override
   public Fields getOutputFields() {
-    if (_proxySpout == null) {
-      // getOutputFields is executed in the topology creation time before serialisation.
-      // do not set the _proxySpout instance variable here to avoid JRuby serialization
-      // issues. Just create tmp spout instance to call getOutputFields.
-      IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-      return spout.getOutputFields();
-    } else {
-      return _proxySpout.getOutputFields();
-    }
+    IRubyObject ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "get_output_fields");
+    return (Fields)ruby_result.toJava(Fields.class);
   }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
-    // getComponentConfiguration is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
-    IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    return spout.getComponentConfiguration();
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // just create tmp spout instance to call getComponentConfiguration.
+
+    IRubyObject ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
-  private static IBatchSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+
+  private IRubyObject initialize_ruby_spout() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.BatchSpout proxy = new redstorm.proxy.BatchSpout(baseClassPath, realSpoutClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }