redstorm 0.6.6 → 0.7.0.beta1

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java

@@ -5,40 +5,28 @@ import backtype.storm.transactional.ITransactionalSpout;
 import backtype.storm.task.TopologyContext;
 import java.util.Map;
 
-/**
- * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
- * this proxy is required to bypass the serialization/deserialization process when dispatching
- * the bolts to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
- *
- * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
- * deserialization at the worker and in the declareOutputFields method which is called once before 
- * serialization at topology creation. 
- */
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 public class JRubyTransactionalCommitterSpout extends JRubyTransactionalSpout implements ICommitterTransactionalSpout {
 
-  ICommitterTransactionalSpout _proxySpout;
-  
   public JRubyTransactionalCommitterSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
     super(baseClassPath, realSpoutClassName, fields);
   }
 
   @Override
   public ICommitterTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
-    // create instance of the jruby class here, after deserialization in the workers.
-    if (_proxySpout == null) {
-      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    }
-    return _proxySpout.getEmitter(conf, context);
-  }
-
-  private static ICommitterTransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
-    try {
-      redstorm.proxy.TransactionalCommitterSpout proxy = new redstorm.proxy.TransactionalCommitterSpout(baseClassPath, realSpoutClassName);
-      return proxy;
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    if (_ruby_spout == null) {
+      IRubyObject _ruby_spout = initialize_ruby_spout();
     }
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "get_emitter", ruby_conf, ruby_context);
+    return (ICommitterTransactionalSpout.Emitter)ruby_result.toJava(ICommitterTransactionalSpout.Emitter.class);
   }
 }

data/src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java

@@ -9,81 +9,104 @@ import backtype.storm.tuple.Tuple;
 import backtype.storm.tuple.Fields;
 import java.util.Map;
 
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
 /**
  * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
  * this proxy is required to bypass the serialization/deserialization process when dispatching
  * the spout to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class). 
+ * (Java serialization call on a JRuby class).
  *
- * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
- * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
- * are called once before serialization at topology creation. 
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which
+ * are called once before serialization at topology creation.
  */
 public class JRubyTransactionalSpout extends BaseTransactionalSpout {
-  ITransactionalSpout _proxySpout;
-  String _realSpoutClassName;
-  String _baseClassPath;
-  String[] _fields;
-  
+  private final String _realSpoutClassName;
+  private final String[] _fields;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  protected transient IRubyObject _ruby_spout;
+  protected transient Ruby __ruby__;
+
   /**
    * create a new JRubySpout
-   * 
-   * @param baseClassPath the topology/project base JRuby class file path 
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
    * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   * @param fields the output fields names
    */
   public JRubyTransactionalSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
-    _baseClassPath = baseClassPath;
     _realSpoutClassName = realSpoutClassName;
     _fields = fields;
+    _bootstrap = "require '" + baseClassPath + "'";
   }
 
   @Override
   public ITransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) {
-    // create instance of the jruby class here, after deserialization in the workers.
-    if (_proxySpout == null) {
-      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    if (_ruby_spout == null) {
+      IRubyObject _ruby_spout = initialize_ruby_spout();
     }
-    return _proxySpout.getCoordinator(conf, context);
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "get_coordinator", ruby_conf, ruby_context);
+    return (ITransactionalSpout.Coordinator)ruby_result.toJava(ITransactionalSpout.Coordinator.class);
   }
 
   @Override
   public ITransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
-    // create instance of the jruby class here, after deserialization in the workers.
-  	if (_proxySpout == null) {
-      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    if (_ruby_spout == null) {
+      IRubyObject _ruby_spout = initialize_ruby_spout();
     }
-    return _proxySpout.getEmitter(conf, context);
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), _ruby_spout, "get_emitter", ruby_conf, ruby_context);
+    return (ITransactionalSpout.Emitter)ruby_result.toJava(ITransactionalSpout.Emitter.class);
   }
 
   @Override
   public void declareOutputFields(OutputFieldsDeclarer declarer) {
-    // declareOutputFields is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // just create tmp spout instance to call declareOutputFields.
+
     if (_fields.length > 0) {
       declarer.declare(new Fields(_fields));
     } else {
-      ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-      spout.declareOutputFields(declarer);
+      IRubyObject ruby_spout = initialize_ruby_spout();
+      IRubyObject ruby_declarer = JavaUtil.convertJavaToRuby(__ruby__, declarer);
+      Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "declare_output_fields", ruby_declarer);
     }
-  }  
+  }
 
   @Override
   public Map<String, Object> getComponentConfiguration() {
-    // getComponentConfiguration is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
-    ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    return spout.getComponentConfiguration();
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // just create tmp spout instance to call getComponentConfiguration.
+
+    IRubyObject ruby_spout = initialize_ruby_spout();
+    IRubyObject ruby_result = Helpers.invoke(__ruby__.getCurrentContext(), ruby_spout, "get_component_configuration");
+    return (Map)ruby_result.toJava(Map.class);
   }
- 
-  private static ITransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+
+  protected IRubyObject initialize_ruby_spout() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
     try {
-      redstorm.proxy.TransactionalSpout proxy = new redstorm.proxy.TransactionalSpout(baseClassPath, realSpoutClassName);
-      return proxy;
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realSpoutClassName);
     }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
   }
 }

data/src/main/redstorm/storm/jruby/JRubyTridentFunction.java

@@ -0,0 +1,66 @@
+package redstorm.storm.jruby;
+
+import storm.trident.tuple.TridentTuple;
+import storm.trident.operation.TridentCollector;
+import java.util.Map;
+import storm.trident.operation.TridentOperationContext;
+import storm.trident.operation.Function;
+
+import org.jruby.Ruby;
+import org.jruby.RubyObject;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.RubyModule;
+import org.jruby.exceptions.RaiseException;
+
+public class JRubyTridentFunction implements Function {
+  private final String _realClassName;
+  private final String _bootstrap;
+
+  // transient to avoid serialization
+  private transient IRubyObject _ruby_function;
+  private transient Ruby __ruby__;
+
+  public JRubyTridentFunction(final String baseClassPath, final String realClassName) {
+    _realClassName = realClassName;
+    _bootstrap = "require '" + baseClassPath + "'";
+  }
+
+  @Override
+  public void execute(final TridentTuple tuple, final TridentCollector collector) {
+    IRubyObject ruby_tuple = JavaUtil.convertJavaToRuby(__ruby__, tuple);
+    IRubyObject ruby_collector = JavaUtil.convertJavaToRuby(__ruby__, collector);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_function, "execute", ruby_tuple, ruby_collector);
+  }
+
+  @Override
+  public void cleanup() {
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_function, "cleanup");
+  }
+
+  @Override
+  public void prepare(final Map conf, final TridentOperationContext context) {
+    if(_ruby_function == null) {
+      _ruby_function = initialize_ruby_function();
+    }
+    IRubyObject ruby_conf = JavaUtil.convertJavaToRuby(__ruby__, conf);
+    IRubyObject ruby_context = JavaUtil.convertJavaToRuby(__ruby__, context);
+    Helpers.invoke(__ruby__.getCurrentContext(), _ruby_function, "prepare", ruby_conf, ruby_context);
+  }
+
+  private IRubyObject initialize_ruby_function() {
+    __ruby__ = Ruby.getGlobalRuntime();
+
+    RubyModule ruby_class;
+    try {
+      ruby_class = __ruby__.getClassFromPath(_realClassName);
+    }
+    catch (RaiseException e) {
+      // after deserialization we need to recreate ruby environment
+      __ruby__.evalScriptlet(_bootstrap);
+      ruby_class = __ruby__.getClassFromPath(_realClassName);
+    }
+    return Helpers.invoke(__ruby__.getCurrentContext(), ruby_class, "new");
+  }
+}

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redstorm
 version: !ruby/object:Gem::Version
-  version: 0.6.6
-  prerelease:
+  version: 0.7.0.beta1
 platform: ruby
 authors:
 - Colin Surprenant
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -18,13 +17,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.13'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.13'
-    none: false
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -34,13 +31,11 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   prerelease: false
   type: :runtime
 description: JRuby integration & DSL for the Storm distributed realtime computation system
@@ -59,19 +54,15 @@ files:
 - lib/red_storm/loggable.rb
 - lib/red_storm/topology_launcher.rb
 - lib/red_storm/version.rb
+- lib/red_storm/dsl/batch_bolt.rb
+- lib/red_storm/dsl/batch_committer_bolt.rb
+- lib/red_storm/dsl/batch_spout.rb
 - lib/red_storm/dsl/bolt.rb
 - lib/red_storm/dsl/drpc_topology.rb
+- lib/red_storm/dsl/output_collector.rb
 - lib/red_storm/dsl/spout.rb
 - lib/red_storm/dsl/topology.rb
 - lib/red_storm/dsl/tuple.rb
-- lib/red_storm/proxy/batch_bolt.rb
-- lib/red_storm/proxy/batch_committer_bolt.rb
-- lib/red_storm/proxy/batch_spout.rb
-- lib/red_storm/proxy/bolt.rb
-- lib/red_storm/proxy/proxy_function.rb
-- lib/red_storm/proxy/spout.rb
-- lib/red_storm/proxy/transactional_committer_spout.rb
-- lib/red_storm/proxy/transactional_spout.rb
 - lib/tasks/red_storm.rake
 - ivy/settings.xml
 - ivy/storm_dependencies.xml
@@ -99,11 +90,12 @@ files:
 - examples/shell/shell_topology.rb
 - examples/shell/resources/splitsentence.py
 - examples/shell/resources/storm.py
+- examples/trident/word_count_query.rb
+- examples/trident/word_count_topology.rb
 - src/main/redstorm/storm/jruby/JRubyBatchBolt.java
 - src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java
 - src/main/redstorm/storm/jruby/JRubyBatchSpout.java
 - src/main/redstorm/storm/jruby/JRubyBolt.java
-- src/main/redstorm/storm/jruby/JRubyProxyFunction.java
 - src/main/redstorm/storm/jruby/JRubyShellBolt.java
 - src/main/redstorm/storm/jruby/JRubyShellSpout.java
 - src/main/redstorm/storm/jruby/JRubySpout.java
@@ -111,6 +103,7 @@ files:
 - src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java
 - src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java
 - src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java
+- src/main/redstorm/storm/jruby/JRubyTridentFunction.java
 - bin/redstorm
 - redstorm.gemspec
 - Rakefile
@@ -118,7 +111,9 @@ files:
 - CHANGELOG.md
 - LICENSE.md
 homepage: https://github.com/colinsurprenant/redstorm
-licenses: []
+licenses:
+- Apache 2.0
+metadata: {}
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -128,17 +123,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
-  none: false
+      version: 1.3.1
 requirements: []
 rubyforge_project: redstorm
-rubygems_version: 1.8.24
+rubygems_version: 2.1.9
 signing_key:
-specification_version: 3
+specification_version: 4
 summary: JRuby on Storm
 test_files: []

data/lib/red_storm/proxy/batch_bolt.rb

@@ -1,63 +0,0 @@
-require 'java'
-
-java_import 'backtype.storm.coordination.BatchOutputCollector'
-java_import 'backtype.storm.task.TopologyContext'
-java_import 'backtype.storm.topology.IRichBolt'
-java_import 'backtype.storm.coordination.IBatchBolt'
-java_import 'backtype.storm.topology.OutputFieldsDeclarer'
-java_import 'backtype.storm.tuple.Tuple'
-java_import 'java.util.Map'
-
-module Backtype
-  java_import 'backtype.storm.Config'
-end
-
-java_package 'redstorm.proxy'
-
-# the Bolt class is a proxy to the real bolt to avoid having to deal with all the
-# Java artifacts when creating a bolt.
-#
-# The real bolt class implementation must define these methods:
-# - prepare(conf, context, collector)
-# - execute(tuple)
-# - declare_output_fields
-#
-# and optionnaly:
-# - cleanup
-#
-class BatchBolt
-  java_implements IBatchBolt
-
-  java_signature 'IBatchBolt (String base_class_path, String real_bolt_class_name)'
-  def initialize(base_class_path, real_bolt_class_name)
-    @real_bolt = Object.module_eval(real_bolt_class_name).new
-  rescue NameError
-    require base_class_path
-    @real_bolt = Object.module_eval(real_bolt_class_name).new
-  end
-
-  java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
-  def prepare(conf, context, collector, id)
-    @real_bolt.prepare(conf, context, collector, id)
-  end
-
-  java_signature 'void execute(Tuple)'
-  def execute(tuple)
-    @real_bolt.execute(tuple)
-  end
-
-  java_signature 'void finishBatch()'
-  def finishBatch
-    @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
-  end
-
-  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
-  def declareOutputFields(declarer)
-    @real_bolt.declare_output_fields(declarer)
-  end
-
-  java_signature 'Map<String, Object> getComponentConfiguration()'
-  def getComponentConfiguration
-    @real_bolt.get_component_configuration
-  end
-end