kb-redstorm 0.6.5 → 0.6.6

data/src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java

@@ -0,0 +1,90 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.transactional.TransactionAttempt;
+import backtype.storm.topology.base.BaseTransactionalBolt;
+import backtype.storm.coordination.BatchOutputCollector;
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalBolt extends BaseTransactionalBolt {
+  IBatchBolt _proxyBolt;
+  String _realBoltClassName;
+  String _baseClassPath; 
+  String[] _fields;
+
+ /**
+   * create a new JRubyBolt
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   */
+  public JRubyTransactionalBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realBoltClassName = realBoltClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final TransactionAttempt id) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    _proxyBolt.prepare(stormConf, context, collector, id);
+  }
+
+  @Override
+  public void execute(Tuple input) {
+    _proxyBolt.execute(input);
+  }
+
+  @Override
+  public void finishBatch() {
+    _proxyBolt.finishBatch();
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java

@@ -0,0 +1,31 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.transactional.TransactionAttempt;
+import backtype.storm.transactional.ICommitter;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalCommitterBolt extends JRubyTransactionalBolt implements ICommitter {
+  public JRubyTransactionalCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    super(baseClassPath, realBoltClassName, fields);
+  }
+
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchCommitterBolt proxy = new redstorm.proxy.BatchCommitterBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java

@@ -0,0 +1,44 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.transactional.ICommitterTransactionalSpout;
+import backtype.storm.transactional.ITransactionalSpout;
+import backtype.storm.task.TopologyContext;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyTransactionalCommitterSpout extends JRubyTransactionalSpout implements ICommitterTransactionalSpout {
+
+  ICommitterTransactionalSpout _proxySpout;
+  
+  public JRubyTransactionalCommitterSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    super(baseClassPath, realSpoutClassName, fields);
+  }
+
+  @Override
+  public ICommitterTransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getEmitter(conf, context);
+  }
+
+  private static ICommitterTransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.TransactionalCommitterSpout proxy = new redstorm.proxy.TransactionalCommitterSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java

@@ -0,0 +1,89 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseTransactionalSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.transactional.ITransactionalSpout;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the spout to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
+ * are called once before serialization at topology creation. 
+ */
+public class JRubyTransactionalSpout extends BaseTransactionalSpout {
+  ITransactionalSpout _proxySpout;
+  String _realSpoutClassName;
+  String _baseClassPath;
+  String[] _fields;
+  
+  /**
+   * create a new JRubySpout
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   */
+  public JRubyTransactionalSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public ITransactionalSpout.Coordinator getCoordinator(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getCoordinator(conf, context);
+  }
+
+  @Override
+  public ITransactionalSpout.Emitter getEmitter(Map conf, TopologyContext context) {
+    // create instance of the jruby class here, after deserialization in the workers.
+  	if (_proxySpout == null) {
+      _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    }
+    return _proxySpout.getEmitter(conf, context);
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      spout.declareOutputFields(declarer);
+    }
+  }  
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    ITransactionalSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+ 
+  private static ITransactionalSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.TransactionalSpout proxy = new redstorm.proxy.TransactionalSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

metadata

@@ -1,79 +1,84 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: kb-redstorm
-version: !ruby/object:Gem::Version
-  prerelease:
-  version: 0.6.5
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 6
+  - 6
+  version: 0.6.6
 platform: ruby
-authors:
+authors: 
 - Colin Surprenant
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-29 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2013-06-11 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: rspec
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.11.0
-    none: false
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.11.0
-    none: false
   prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 11
+        - 0
+        version: 2.11.0
   type: :development
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: !binary |-
-          MA==
-    none: false
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: !binary |-
-          MA==
-    none: false
   prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
+  version_requirements: *id002
 description: JRuby integration & DSL for the Storm distributed realtime computation system
-email:
+email: 
 - colin.surprenant@gmail.com
-executables:
+executables: 
 - redstorm
 extensions: []
+
 extra_rdoc_files: []
-files:
-- lib/red_storm.rb
+
+files: 
 - lib/red_storm/application.rb
 - lib/red_storm/configuration.rb
 - lib/red_storm/configurator.rb
 - lib/red_storm/environment.rb
 - lib/red_storm/loggable.rb
+- lib/red_storm/proxy/batch_bolt.rb
+- lib/red_storm/proxy/batch_committer_bolt.rb
+- lib/red_storm/proxy/batch_spout.rb
+- lib/red_storm/proxy/bolt.rb
+- lib/red_storm/proxy/proxy_function.rb
+- lib/red_storm/proxy/spout.rb
+- lib/red_storm/proxy/transactional_committer_spout.rb
+- lib/red_storm/proxy/transactional_spout.rb
 - lib/red_storm/simple_bolt.rb
 - lib/red_storm/simple_drpc_topology.rb
 - lib/red_storm/simple_spout.rb
 - lib/red_storm/simple_topology.rb
 - lib/red_storm/topology_launcher.rb
 - lib/red_storm/version.rb
-- lib/red_storm/proxy/batch_spout.rb
-- lib/red_storm/proxy/bolt.rb
-- lib/red_storm/proxy/proxy_function.rb
-- lib/red_storm/proxy/spout.rb
+- lib/red_storm.rb
 - lib/tasks/red_storm.rake
 - ivy/settings.xml
+- ivy/storm_dependencies.xml
+- ivy/topology_dependencies.xml
 - examples/native/cluster_word_count_topology.rb
 - examples/native/exclamation_bolt.rb
-- examples/native/Gemfile
 - examples/native/local_exclamation_topology.rb
 - examples/native/local_exclamation_topology2.rb
 - examples/native/local_redis_word_count_topology.rb
@@ -81,12 +86,13 @@ files:
 - examples/native/random_sentence_spout.rb
 - examples/native/split_sentence_bolt.rb
 - examples/native/word_count_bolt.rb
-- examples/shell/shell_topology.rb
 - examples/shell/resources/splitsentence.py
 - examples/shell/resources/storm.py
+- examples/shell/shell_topology.rb
 - examples/simple/exclamation_bolt.rb
 - examples/simple/exclamation_topology.rb
 - examples/simple/exclamation_topology2.rb
+- examples/simple/hello_world_topology.rb
 - examples/simple/kafka_topology.rb
 - examples/simple/random_sentence_spout.rb
 - examples/simple/redis_word_count_topology.rb
@@ -94,41 +100,53 @@ files:
 - examples/simple/split_sentence_bolt.rb
 - examples/simple/word_count_bolt.rb
 - examples/simple/word_count_topology.rb
+- src/main/redstorm/storm/jruby/JRubyBatchBolt.java
+- src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java
 - src/main/redstorm/storm/jruby/JRubyBatchSpout.java
 - src/main/redstorm/storm/jruby/JRubyBolt.java
 - src/main/redstorm/storm/jruby/JRubyProxyFunction.java
 - src/main/redstorm/storm/jruby/JRubyShellBolt.java
 - src/main/redstorm/storm/jruby/JRubyShellSpout.java
 - src/main/redstorm/storm/jruby/JRubySpout.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalBolt.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalCommitterBolt.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalCommitterSpout.java
+- src/main/redstorm/storm/jruby/JRubyTransactionalSpout.java
 - bin/redstorm
+- redstorm.gemspec
 - Rakefile
 - README.md
 - CHANGELOG.md
 - LICENSE.md
+has_rdoc: true
 homepage: https://github.com/colinsurprenant/redstorm
 licenses: []
-post_install_message:
+
+post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
-  requirements:
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
   - - ">="
-    - !ruby/object:Gem::Version
-      version: !binary |-
-        MA==
-  none: false
-required_rubygems_version: !ruby/object:Gem::Requirement
-  requirements:
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
   - - ">="
-    - !ruby/object:Gem::Version
-      version: !binary |-
-        MA==
-  none: false
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: redstorm
-rubygems_version: 1.8.24
-signing_key:
+rubygems_version: 1.3.6
+signing_key: 
 specification_version: 3
 summary: JRuby on Storm
 test_files: []
+

data/examples/native/Gemfile

@@ -1,2 +0,0 @@
-source :rubygems
-gem 'redis'