kb-redstorm 0.6.4

data/src/main/redstorm/storm/jruby/JRubyProxyFunction.java

@@ -0,0 +1,59 @@
+package redstorm.storm.jruby;
+
+import storm.trident.tuple.TridentTuple;
+import storm.trident.operation.TridentCollector;
+import java.util.Map;
+import storm.trident.operation.TridentOperationContext;
+import storm.trident.operation.Function;
+
+public class JRubyProxyFunction implements Function {
+  Function _proxy;
+  String _realClassName;
+  String _baseClassPath;
+  String[] _fields;
+
+  public JRubyProxyFunction(final String baseClassPath, final String realClassName, final String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realClassName = realClassName;
+    _fields = fields;
+  }
+
+
+  @Override
+  public void execute(final TridentTuple _tridentTuple, final TridentCollector _tridentCollector) {
+    
+    if(_proxy == null) {
+      _proxy = newProxy(_baseClassPath, _realClassName);
+    }
+    _proxy.execute(_tridentTuple, _tridentCollector);
+    
+  }
+
+  @Override
+  public void cleanup() {
+    
+    _proxy.cleanup();
+    
+  }
+
+  @Override
+  public void prepare(final Map _map, final TridentOperationContext _tridentOperationContext) {
+    
+    if(_proxy == null) {
+      _proxy = newProxy(_baseClassPath, _realClassName);
+    }
+    _proxy.prepare(_map, _tridentOperationContext);
+    
+  }
+
+
+  private static Function newProxy(final String baseClassPath, final String realClassName) {
+    try {
+      redstorm.proxy.ProxyFunction proxy = new redstorm.proxy.ProxyFunction(baseClassPath, realClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyShellBolt.java

@@ -0,0 +1,26 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.ShellBolt;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+    
+public class JRubyShellBolt  extends ShellBolt  implements IRichBolt {
+  private String[] _fields;
+
+  public JRubyShellBolt(String[] command, String[] fields) {
+    super(command);
+    _fields = fields;
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declare(new Fields(_fields));
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    return null;
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyShellSpout.java

@@ -0,0 +1,26 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.spout.ShellSpout;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+public class JRubyShellSpout extends ShellSpout implements IRichSpout {
+  private String[] _fields;
+
+  public JRubyShellSpout(String[] command, String[] fields) {
+    super(command);
+    _fields = fields;
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declare(new Fields(_fields));
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    return null;
+  }
+}

data/src/main/redstorm/storm/jruby/JRubySpout.java

@@ -0,0 +1,107 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the spout to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
+ * are called once before serialization at topology creation. 
+ */
+public class JRubySpout implements IRichSpout {
+  IRichSpout _proxySpout;
+  String _realSpoutClassName;
+  String _baseClassPath;
+  String[] _fields;
+
+  /**
+   * create a new JRubySpout
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   */
+  public JRubySpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
+    // create instance of the jruby proxy class here, after deserialization in the workers.
+    _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    _proxySpout.open(conf, context, collector);
+  }
+
+  @Override
+  public void close() {
+    _proxySpout.close();
+  }
+
+  @Override
+  public void activate() {
+    _proxySpout.activate();
+  }
+
+  @Override
+  public void deactivate() {
+    _proxySpout.deactivate();
+  }
+
+  @Override
+  public void nextTuple() {
+    _proxySpout.nextTuple();
+  }
+
+  @Override
+  public void ack(Object msgId) {
+    _proxySpout.ack(msgId);
+  }
+
+  @Override
+  public void fail(Object msgId) {
+    _proxySpout.fail(msgId);
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      spout.declareOutputFields(declarer);
+    }
+  }  
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+ 
+  private static IRichSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.Spout proxy = new redstorm.proxy.Spout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: kb-redstorm
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 0.6.4
+platform: ruby
+authors:
+- Colin Surprenant
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-04-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  prerelease: false
+  type: :runtime
+description: JRuby integration & DSL for the Storm distributed realtime computation system
+email:
+- colin.surprenant@gmail.com
+executables:
+- redstorm
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/red_storm.rb
+- lib/red_storm/application.rb
+- lib/red_storm/configuration.rb
+- lib/red_storm/configurator.rb
+- lib/red_storm/environment.rb
+- lib/red_storm/loggable.rb
+- lib/red_storm/simple_bolt.rb
+- lib/red_storm/simple_drpc_topology.rb
+- lib/red_storm/simple_spout.rb
+- lib/red_storm/simple_topology.rb
+- lib/red_storm/topology_launcher.rb
+- lib/red_storm/version.rb
+- lib/red_storm/proxy/batch_spout.rb
+- lib/red_storm/proxy/bolt.rb
+- lib/red_storm/proxy/proxy_function.rb
+- lib/red_storm/proxy/spout.rb
+- lib/tasks/red_storm.rake
+- ivy/settings.xml
+- examples/native/cluster_word_count_topology.rb
+- examples/native/exclamation_bolt.rb
+- examples/native/Gemfile
+- examples/native/local_exclamation_topology.rb
+- examples/native/local_exclamation_topology2.rb
+- examples/native/local_redis_word_count_topology.rb
+- examples/native/local_word_count_topology.rb
+- examples/native/random_sentence_spout.rb
+- examples/native/split_sentence_bolt.rb
+- examples/native/word_count_bolt.rb
+- examples/shell/shell_topology.rb
+- examples/shell/resources/splitsentence.py
+- examples/shell/resources/storm.py
+- examples/simple/exclamation_bolt.rb
+- examples/simple/exclamation_topology.rb
+- examples/simple/exclamation_topology2.rb
+- examples/simple/kafka_topology.rb
+- examples/simple/random_sentence_spout.rb
+- examples/simple/redis_word_count_topology.rb
+- examples/simple/ruby_version_topology.rb
+- examples/simple/split_sentence_bolt.rb
+- examples/simple/word_count_bolt.rb
+- examples/simple/word_count_topology.rb
+- src/main/redstorm/storm/jruby/JRubyBatchSpout.java
+- src/main/redstorm/storm/jruby/JRubyBolt.java
+- src/main/redstorm/storm/jruby/JRubyProxyFunction.java
+- src/main/redstorm/storm/jruby/JRubyShellBolt.java
+- src/main/redstorm/storm/jruby/JRubyShellSpout.java
+- src/main/redstorm/storm/jruby/JRubySpout.java
+- bin/redstorm
+- Rakefile
+- README.md
+- CHANGELOG.md
+- LICENSE.md
+homepage: https://github.com/colinsurprenant/redstorm
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+requirements: []
+rubyforge_project: redstorm
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: JRuby on Storm
+test_files: []