kb-redstorm 0.6.5 → 0.6.6

data/lib/red_storm/simple_topology.rb

@@ -106,6 +106,7 @@ module RedStorm
 
     # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
     def self.spout(spout_class, *args, &spout_block)
+      set_topology_class!
       options = args.last.is_a?(Hash) ? args.pop : {}
       contructor_args = !args.empty? ? args.pop : []
       spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
@@ -117,9 +118,10 @@ module RedStorm
 
     # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
     def self.bolt(bolt_class, *args, &bolt_block)
+      set_topology_class!
       options = args.last.is_a?(Hash) ? args.pop : {}
       contructor_args = !args.empty? ? args.pop : []
-      bolt_options = {:id => options[:id] ? options[:id] : self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
 
       bolt = BoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
       raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
@@ -128,8 +130,8 @@ module RedStorm
     end
 
     def self.configure(name = nil, &configure_block)
-      Configuration.topology_class = self
-      @topology_name = name if name
+      set_topology_class!
+      @topology_name = name.to_s if name
       @configure_block = configure_block if block_given?
     end
 
@@ -158,7 +160,7 @@ module RedStorm
 
       configurator = Configurator.new(defaults)
       configurator.instance_exec(env, &self.class.configure_block)
-
+ 
       submitter = (env == :local) ? @cluster = LocalCluster.new : StormSubmitter
       submitter.submitTopology(self.class.topology_name, configurator.config, builder.createTopology)
       instance_exec(env, &self.class.submit_block)
@@ -166,6 +168,14 @@ module RedStorm
 
     private
 
+    # this is a quirk to figure out the topology class at load time when the topology file
+    # is required in the TopologyLauncher. Since we want to make the "configure" DSL statement 
+    # optional we can hook into any/all the other DSL statements that will be called at load time
+    # and set it there. This is somewhat inelegant but it works. 
+    def self.set_topology_class!
+      Configuration.topology_class = self
+    end
+
     def self.resolve_ids!(components)
       # verify duplicate implicit ids
       ids = components.map(&:id)

data/lib/red_storm/topology_launcher.rb

@@ -1,5 +1,10 @@
 require 'java'
 
+# This hack get rif of the "Use RbConfig instead of obsolete and deprecated Config"
+# deprecation warning that is triggered by "java_import 'backtype.storm.Config'". 
+Object.send :remove_const, :Config
+Config = RbConfig
+
 # see https://github.com/colinsurprenant/redstorm/issues/7
 module Backtype
   java_import 'backtype.storm.Config'
@@ -9,6 +14,7 @@ java_import 'backtype.storm.LocalCluster'
 java_import 'backtype.storm.LocalDRPC'
 java_import 'backtype.storm.StormSubmitter'
 java_import 'backtype.storm.topology.TopologyBuilder'
+java_import 'backtype.storm.coordination.BatchBoltExecutor'
 java_import 'backtype.storm.drpc.LinearDRPCTopologyBuilder'
 java_import 'backtype.storm.tuple.Fields'
 java_import 'backtype.storm.tuple.Tuple'
@@ -16,7 +22,12 @@ java_import 'backtype.storm.tuple.Values'
 
 java_import 'redstorm.storm.jruby.JRubyBolt'
 java_import 'redstorm.storm.jruby.JRubySpout'
+java_import 'redstorm.storm.jruby.JRubyBatchBolt'
+java_import 'redstorm.storm.jruby.JRubyBatchCommitterBolt'
 java_import 'redstorm.storm.jruby.JRubyBatchSpout'
+java_import 'redstorm.storm.jruby.JRubyTransactionalSpout'
+java_import 'redstorm.storm.jruby.JRubyTransactionalBolt'
+java_import 'redstorm.storm.jruby.JRubyTransactionalCommitterBolt'
 
 java_package 'redstorm'
 
@@ -41,6 +52,11 @@ class TopologyLauncher
 
     require "#{class_path}"
 
+    if RedStorm::Configuration.topology_class.nil? || !RedStorm::Configuration.topology_class.method_defined?(:start)
+      puts("\nERROR: invalid topology class. make sure your topology class is a subclass of one of the DSL topology classes or that your class sets RedStorm::Configuration.topology_class and defines the start method\n\n")
+      exit(1)
+    end
+
     topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
     puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
     RedStorm::Configuration.topology_class.new.start(class_path, env)

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.6.5'
+  VERSION = '0.6.6'
 end

data/lib/tasks/red_storm.rake

@@ -1,8 +1,7 @@
 begin
   require 'ant'
 rescue
-  puts("error: unable to load Ant, make sure Ant is installed, in your PATH")
-  puts("       and $ANT_HOME is defined properly.")
+  puts("ERROR: unable to load Ant, make sure Ant is installed, in your PATH and $ANT_HOME is defined properly")
   puts("\nerror detail:\n#{$!}")
   exit(1)
 end
@@ -11,29 +10,18 @@ require 'jruby/jrubyc'
 require 'red_storm'
 require 'red_storm/application'
 
-DEP_STORM_VERSION = "0.8.2"
-DEP_JRUBY_VERSION = "1.7.3"
-INSTALL_IVY_VERSION = "2.2.0"
-
-DEFAULT_DEPENDENCIES = {
-  :storm_artifacts => [
-    "storm:storm:#{DEP_STORM_VERSION}, transitive=true",
-  ],
-  :topology_artifacts => [
-    "org.jruby:jruby-complete:#{DEP_JRUBY_VERSION}, transitive=false",
-  ],
-}
+INSTALL_IVY_VERSION = "2.3.0"
 
 task :launch, :env, :ruby_mode, :class_file do |t, args|
   # use ruby mode parameter or default to current interpreter version
   version_token = RedStorm.jruby_mode_token(args[:ruby_mode])
-
+  
   command = case args[:env]
   when "local"
     RedStorm::Application.local_storm_command(args[:class_file], args[:ruby_mode])
   when "cluster"
     unless File.exist?(TARGET_CLUSTER_JAR)
-      puts("error: cluster jar file #{TARGET_CLUSTER_JAR} not found. Generate it usingw $redstorm jar DIR1 [DIR2, ...]")
+      puts("error: cluster jar file #{TARGET_CLUSTER_JAR} not found. Generate it using $redstorm jar DIR1 [DIR2, ...]")
       exit(1)
     end
     RedStorm::Application.cluster_storm_command(args[:class_file], args[:ruby_mode])
@@ -46,25 +34,25 @@ task :launch, :env, :ruby_mode, :class_file do |t, args|
 end
 
 task :clean do
-  ant.delete :dir => TARGET_DIR
+  ant.delete 'dir' => TARGET_DIR
 end
 
 task :clean_jar do
-  ant.delete :file => TARGET_CLUSTER_JAR
+  ant.delete 'file' => TARGET_CLUSTER_JAR
 end
 
 task :setup do
   puts("\n--> Setting up target directories")
-  ant.mkdir :dir => TARGET_DIR
-  ant.mkdir :dir => TARGET_CLASSES_DIR
-  ant.mkdir :dir => TARGET_DEPENDENCY_DIR
-  ant.mkdir :dir => TARGET_SRC_DIR
-  ant.mkdir :dir => TARGET_GEM_DIR
-  ant.mkdir :dir => TARGET_SPECS_DIR
-  ant.path :id => 'classpath' do
-    fileset :dir => TARGET_DEPENDENCY_DIR
-    fileset :dir => TARGET_CLASSES_DIR
-  end
+  ant.mkdir 'dir' => TARGET_DIR 
+  ant.mkdir 'dir' => TARGET_CLASSES_DIR 
+  ant.mkdir 'dir' => TARGET_DEPENDENCY_DIR
+  ant.mkdir 'dir' => TARGET_SRC_DIR
+  ant.mkdir 'dir' => TARGET_GEM_DIR
+  ant.mkdir 'dir' => TARGET_SPECS_DIR
+  ant.path 'id' => 'classpath' do  
+    fileset 'dir' => TARGET_DEPENDENCY_DIR  
+    fileset 'dir' => TARGET_CLASSES_DIR  
+  end  
 end
 
 task :install => [:deps, :build] do
@@ -112,18 +100,23 @@ task :bundle, [:groups] => :setup do |t, args|
   defaulted_args = {:groups => 'default'}.merge(args.to_hash.delete_if{|k, v| v.to_s.empty?})
   groups = defaulted_args[:groups].split(':').map(&:to_sym)
   Bundler.definition.specs_for(groups).each do |spec|
-    unless spec.full_name =~ /^bundler-\d+/
-      destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"
-      unless File.directory?(destination_path)
-        puts("installing gem #{spec.full_name} into #{destination_path}")
-        # copy the actual gem dir
-        FileUtils.cp_r(spec.full_gem_path, destination_path)
-        # copy the gemspec into the specifications/ dir
-        FileUtils.cp_r(spec.loaded_from, TARGET_SPECS_DIR)
-        # strip the .git directory from git dependencies, it can be huge
-        FileUtils.rm_rf("#{destination_path}/.git")
-      end
-    end
+    next if spec.name == 'bundler'
+
+    # try to avoid infinite recursion
+    next if TARGET_GEM_DIR.start_with?(spec.full_gem_path)
+
+    destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"
+    next if File.directory?(destination_path)
+
+    puts("installing gem #{spec.full_name} into #{destination_path}")
+    # copy the actual gem dir
+    FileUtils.cp_r(spec.full_gem_path, destination_path)
+    # copy the evaluated gemspec into the specifications/ dir (we
+    # may not have enough info to reconstruct once we delete the
+    # .git directory)
+    File.open(File.join(TARGET_SPECS_DIR, File.basename(spec.loaded_from)), 'w'){|f| f.write(spec.to_ruby)}
+    # strip the .git directory from git dependencies, it can be huge
+    FileUtils.rm_rf("#{destination_path}/.git")
   end
 end
 
@@ -131,21 +124,21 @@ namespace :ivy do
   task :download do
     mkdir_p DST_IVY_DIR
     ant.get({
-      :src => "http://repo1.maven.org/maven2/org/apache/ivy/ivy/#{INSTALL_IVY_VERSION}/ivy-#{INSTALL_IVY_VERSION}.jar",
-      :dest => "#{DST_IVY_DIR}/ivy-#{INSTALL_IVY_VERSION}.jar",
-      :usetimestamp => true,
+      'src' => "http://repo1.maven.org/maven2/org/apache/ivy/ivy/#{INSTALL_IVY_VERSION}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      'dest' => "#{DST_IVY_DIR}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      'usetimestamp' => true,
     })
   end
 
   task :install => :download do
-    ant.path :id => 'ivy.lib.path' do
-      fileset :dir => DST_IVY_DIR, :includes => '*.jar'
+    ant.path 'id' => 'ivy.lib.path' do
+      fileset 'dir' => DST_IVY_DIR, 'includes' => '*.jar'
     end
 
     ant.taskdef({
-      :resource => "org/apache/ivy/ant/antlib.xml",
-      :classpathref => "ivy.lib.path",
-      #:uri => "antlib:org.apache.ivy.ant",
+      'resource' => "org/apache/ivy/ant/antlib.xml",
+      'classpathref' => "ivy.lib.path",
+      #'uri' => "antlib:org.apache.ivy.ant",
     })
   end
 end
@@ -153,37 +146,32 @@ end
 task :deps => "ivy:install" do
   puts("\n--> Installing dependencies")
 
-  dependencies = File.exists?(CUSTOM_DEPENDENCIES) ? eval(File.read(CUSTOM_DEPENDENCIES)) : DEFAULT_DEPENDENCIES
-  ant.configure :file => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
+  ant.configure 'file' => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
 
-  dependencies[:storm_artifacts].each do |dependency|
-    artifact, transitive = dependency.split(/\s*,\s*/)
-    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/storm", "default"]))
-  end
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_STORM_DEPENDENCIES) ? CUSTOM_IVY_STORM_DEPENDENCIES : DEFAULT_IVY_STORM_DEPENDENCIES 
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/storm/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"  
 
-  dependencies[:topology_artifacts].each do |dependency|
-    artifact, transitive = dependency.split(/\s*,\s*/)
-    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/topology", "default"]))
-  end
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_TOPOLOGY_DEPENDENCIES) ? CUSTOM_IVY_TOPOLOGY_DEPENDENCIES : DEFAULT_IVY_TOPOLOGY_DEPENDENCIES 
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/topology/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"
 end
 
 task :jar, [:include_dir] => [:clean_jar] do |t, args|
   puts("\n--> Generating JAR file #{TARGET_CLUSTER_JAR}")
 
-  ant.jar :destfile => TARGET_CLUSTER_JAR do
+  ant.jar 'destfile' => TARGET_CLUSTER_JAR do
     # rejar all topology jars
     Dir["target/dependency/topology/default/*.jar"].each do |jar|
       puts("Extracting #{jar}")
-      zipfileset :src => jar, :includes => "**/*"
+      zipfileset 'src' => jar, 'includes' => "**/*"
     end
-    fileset :dir => TARGET_DIR do
-      include :name => "gems/**"
+    fileset 'dir' => TARGET_DIR do
+      include 'name' => "gems/**"
     end
-    fileset :dir => TARGET_CLASSES_DIR
+    fileset 'dir' => TARGET_CLASSES_DIR
     # red_storm.rb and red_storm/* must be in root of jar so that "require 'red_storm'"
     # in bolts/spouts works in jar context
-    fileset :dir => TARGET_LIB_DIR do
-      exclude :name => "tasks/**"
+    fileset 'dir' => TARGET_LIB_DIR do
+      exclude 'name' => "tasks/**"
     end
     if args[:include_dir]
       dirs = args[:include_dir].split(":")
@@ -193,19 +181,19 @@ task :jar, [:include_dir] => [:clean_jar] do |t, args|
         resources_dirs = Dir.glob("#{dir}/**/resources")
         resources_dirs.each do |resources_dir|
           resources_parent = resources_dir.gsub("/resources", "")
-          fileset :dir => resources_parent do
-            include :name => "resources/**/*"
+          fileset 'dir' => resources_parent do
+            include 'name' => "resources/**/*"
           end
         end
       end
 
       # include complete source dir tree (note we don't care about potential duplicated resources dir)
-      fileset :dir => CWD do
-        dirs.each{|dir| include :name => "#{dir}/**/*"}
+      fileset 'dir' => CWD do
+        dirs.each{|dir| include 'name' => "#{dir}/**/*"}
       end
     end
     manifest do
-      attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
+      attribute 'name' => "Main-Class", 'value' => "redstorm.TopologyLauncher"
     end
   end
   puts("\nRedStorm generated JAR file #{TARGET_CLUSTER_JAR}")
@@ -214,19 +202,19 @@ end
 def build_java_dir(source_folder)
   puts("\n--> Compiling Java")
   ant.javac(
-    :srcdir => source_folder,
-    :destdir => TARGET_CLASSES_DIR,
-    :classpathref => 'classpath',
-    :source => "1.6",
-    :target => "1.6",
-    :debug => "yes",
-    :includeantruntime => "no",
-    :verbose => false,
-    :listfiles => true
+    'srcdir' => source_folder,
+    'destdir' => TARGET_CLASSES_DIR,
+    'classpathref' => 'classpath', 
+    'source' => "1.6",
+    'target' => "1.6",
+    'debug' => "yes",
+    'includeantruntime' => "no",
+    'verbose' => false,
+    'listfiles' => true
   ) do
     # compilerarg :value => "-Xlint:unchecked"
-  end
-end
+  end 
+end  
 
 def build_jruby(source_path)
   puts("\n--> Compiling JRuby")
@@ -243,30 +231,3 @@ def build_jruby(source_path)
     status =  JRuby::Compiler::compile_argv(argv)
   end
 end
-
-def truefalse(s)
-  return true if s.to_s.downcase =~ /1|yes|true/
-  return false if s.to_s.downcase =~ /0|no|false/
-  nil
-end
-
-def ivy_retrieve(org, mod, rev, transitive, dir, conf)
-  ant.resolve({
-    :organisation => org,
-    :module => mod,
-    :revision => rev,
-    :inline => true,
-    :transitive => truefalse(transitive),
-    :conf => conf,
-  })
-
-  ant.retrieve({
-    :organisation => org,
-    :module => mod,
-    :revision => rev,
-    :pattern => "#{dir}/[conf]/[artifact]-[revision].[ext]",
-    :inline => true,
-    :transitive => truefalse(transitive),
-    :conf => conf,
-  })
-end

data/redstorm.gemspec

@@ -0,0 +1,24 @@
+libdir = File.expand_path('../lib/', __FILE__)
+$:.unshift libdir unless $:.include?(libdir)
+
+require 'red_storm/version'
+
+Gem::Specification.new do |s|  
+  s.name        = 'kb-redstorm'
+  s.version     = RedStorm::VERSION
+  s.authors     = ['Colin Surprenant']
+  s.email       = ['colin.surprenant@gmail.com']
+  s.homepage    = 'https://github.com/colinsurprenant/redstorm'
+  s.summary     = 'JRuby on Storm'
+  s.description = 'JRuby integration & DSL for the Storm distributed realtime computation system'
+ 
+  s.rubyforge_project = 'redstorm'
+  
+  s.files         = Dir.glob("{lib/**/*}") + Dir.glob("{ivy/*.xml}") + Dir.glob("{examples/**/*}") + Dir.glob("{src/**/*.java}") + Dir.glob("{bin/**/*}") + %w(redstorm.gemspec Rakefile README.md CHANGELOG.md LICENSE.md)
+  s.require_paths = ['lib']
+  s.bindir        = 'bin'
+  s.executables   = ['redstorm']
+
+  s.add_development_dependency 'rspec', '~> 2.11.0'
+  s.add_runtime_dependency 'rake'
+end

data/src/main/redstorm/storm/jruby/JRubyBatchBolt.java

@@ -0,0 +1,90 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseBatchBolt;
+import backtype.storm.coordination.BatchOutputCollector;
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyBatchBolt extends BaseBatchBolt {
+  IBatchBolt _proxyBolt;
+  String _realBoltClassName;
+  String _baseClassPath; 
+  String[] _fields;
+
+  /**
+   * create a new JRubyBolt
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   */
+  public JRubyBatchBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realBoltClassName = realBoltClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final Object id) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    _proxyBolt.prepare(stormConf, context, collector, id);
+  }
+
+  @Override
+  public void execute(Tuple input) {
+    _proxyBolt.execute(input);
+  }
+
+  @Override
+  public void finishBatch() {
+    _proxyBolt.finishBatch();
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java

@@ -0,0 +1,9 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.transactional.ICommitter;
+
+public class JRubyBatchCommitterBolt extends JRubyBatchBolt implements ICommitter {
+  public JRubyBatchCommitterBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    super(baseClassPath, realBoltClassName, fields);
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java

@@ -1,39 +1,35 @@
 package redstorm.storm.jruby;
 
-import storm.trident.operation.TridentCollector;
+import backtype.storm.tuple.Fields;
 import backtype.storm.task.TopologyContext;
+import storm.trident.operation.TridentCollector;
 import storm.trident.spout.IBatchSpout;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Fields;
 import java.util.Map;
 
 /**
- * the JRubyBatchSpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * the JRubySpout class is a simple proxy class to the actual spout implementation in JRuby.
  * this proxy is required to bypass the serialization/deserialization process when dispatching
  * the spout to the workers. JRuby does not yet support serialization from Java
- * (Java serialization call on a JRuby class).
+ * (Java serialization call on a JRuby class). 
  *
- * Note that the JRuby spout proxy class is instanciated in the open method which is called after
- * deserialization at the worker and in both the declareOutputFields and isDistributed methods which
- * are called once before serialization at topology creation.
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after 
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which 
+ * are called once before serialization at topology creation. 
  */
 public class JRubyBatchSpout implements IBatchSpout {
   IBatchSpout _proxySpout;
   String _realSpoutClassName;
   String _baseClassPath;
-  String[] _fields;
-
+  
   /**
-   * create a new JRubyBatchSpout
-   *
-   * @param baseClassPath the topology/project base JRuby class file path
+   * create a new JRubySpout
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
    * @param realSpoutClassName the fully qualified JRuby spout implementation class name
    */
-  public JRubyBatchSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+  public JRubyBatchSpout(String baseClassPath, String realSpoutClassName) {
     _baseClassPath = baseClassPath;
     _realSpoutClassName = realSpoutClassName;
-    _fields = fields;
   }
 
   @Override
@@ -44,10 +40,9 @@ public class JRubyBatchSpout implements IBatchSpout {
   }
 
   @Override
-  public void emitBatch(final long batchId, final TridentCollector collector) {
+  public void emitBatch(long batchId, TridentCollector collector) {
     _proxySpout.emitBatch(batchId, collector);
-  }
-
+  } 
 
   @Override
   public void close() {
@@ -55,17 +50,21 @@ public class JRubyBatchSpout implements IBatchSpout {
   }
 
   @Override
-  public void ack(final long batchId) {
+  public void ack(long batchId) {
     _proxySpout.ack(batchId);
   }
 
   @Override
   public Fields getOutputFields() {
-    // getOutputFields is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call declareOutputFields.
-    IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    return spout.getOutputFields();
+    if (_proxySpout == null) {
+      // getOutputFields is executed in the topology creation time before serialisation.
+      // do not set the _proxySpout instance variable here to avoid JRuby serialization
+      // issues. Just create tmp spout instance to call getOutputFields.
+      IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      return spout.getOutputFields();
+    } else {
+      return _proxySpout.getOutputFields();
+    }
   }
 
   @Override
@@ -76,7 +75,7 @@ public class JRubyBatchSpout implements IBatchSpout {
     IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
     return spout.getComponentConfiguration();
   }
-
+ 
   private static IBatchSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
     try {
       redstorm.proxy.BatchSpout proxy = new redstorm.proxy.BatchSpout(baseClassPath, realSpoutClassName);

data/src/main/redstorm/storm/jruby/JRubyProxyFunction.java

@@ -10,40 +10,32 @@ public class JRubyProxyFunction implements Function {
   Function _proxy;
   String _realClassName;
   String _baseClassPath;
-  String[] _fields;
 
-  public JRubyProxyFunction(final String baseClassPath, final String realClassName, final String[] fields) {
+  public JRubyProxyFunction(final String baseClassPath, final String realClassName) {
     _baseClassPath = baseClassPath;
     _realClassName = realClassName;
-    _fields = fields;
   }
 
 
   @Override
   public void execute(final TridentTuple _tridentTuple, final TridentCollector _tridentCollector) {
-    
     if(_proxy == null) {
       _proxy = newProxy(_baseClassPath, _realClassName);
     }
     _proxy.execute(_tridentTuple, _tridentCollector);
-    
   }
 
   @Override
   public void cleanup() {
-    
     _proxy.cleanup();
-    
   }
 
   @Override
   public void prepare(final Map _map, final TridentOperationContext _tridentOperationContext) {
-    
     if(_proxy == null) {
       _proxy = newProxy(_baseClassPath, _realClassName);
     }
     _proxy.prepare(_map, _tridentOperationContext);
-    
   }