redstorm 0.6.4 → 0.6.5

data/lib/red_storm/topology_launcher.rb

@@ -1,23 +1,37 @@
 require 'java'
 
+# This hack get rif of the "Use RbConfig instead of obsolete and deprecated Config"
+# deprecation warning that is triggered by "java_import 'backtype.storm.Config'". 
+Object.send :remove_const, :Config
+Config = RbConfig
+
 # see https://github.com/colinsurprenant/redstorm/issues/7
 module Backtype
   java_import 'backtype.storm.Config'
 end
 
 java_import 'backtype.storm.LocalCluster'
+java_import 'backtype.storm.LocalDRPC'
 java_import 'backtype.storm.StormSubmitter'
 java_import 'backtype.storm.topology.TopologyBuilder'
+java_import 'backtype.storm.coordination.BatchBoltExecutor'
+java_import 'backtype.storm.drpc.LinearDRPCTopologyBuilder'
 java_import 'backtype.storm.tuple.Fields'
 java_import 'backtype.storm.tuple.Tuple'
 java_import 'backtype.storm.tuple.Values'
 
 java_import 'redstorm.storm.jruby.JRubyBolt'
 java_import 'redstorm.storm.jruby.JRubySpout'
+java_import 'redstorm.storm.jruby.JRubyBatchBolt'
+java_import 'redstorm.storm.jruby.JRubyBatchCommitterBolt'
+java_import 'redstorm.storm.jruby.JRubyBatchSpout'
+java_import 'redstorm.storm.jruby.JRubyTransactionalSpout'
+java_import 'redstorm.storm.jruby.JRubyTransactionalBolt'
+java_import 'redstorm.storm.jruby.JRubyTransactionalCommitterBolt'
 
 java_package 'redstorm'
 
-# TopologyLauncher is the application entry point when launching a topology. Basically it will 
+# TopologyLauncher is the application entry point when launching a topology. Basically it will
 # call require on the specified Ruby topology class file path and call its start method
 class TopologyLauncher
 
@@ -36,14 +50,19 @@ class TopologyLauncher
     $:.unshift File.expand_path(launch_path + '/lib')
     $:.unshift File.expand_path(launch_path + '/target/lib')
 
-    require "#{class_path}" 
+    require "#{class_path}"
+
+    if RedStorm::Configuration.topology_class.nil? || !RedStorm::Configuration.topology_class.method_defined?(:start)
+      puts("\nERROR: invalid topology class. make sure your topology class is a subclass of one of the DSL topology classes or that your class sets RedStorm::Configuration.topology_class and defines the start method\n\n")
+      exit(1)
+    end
 
     topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
     puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
     RedStorm::Configuration.topology_class.new.start(class_path, env)
   end
 
-  private 
+  private
 
   def self.camel_case(s)
     s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.6.4'
+  VERSION = '0.6.5'
 end

data/lib/tasks/red_storm.rake

@@ -1,7 +1,7 @@
 begin
   require 'ant'
 rescue
-  puts("error: unable to load Ant, make sure Ant is installed and in your PATH")
+  puts("ERROR: unable to load Ant, make sure Ant is installed, in your PATH and $ANT_HOME is defined properly")
   puts("\nerror detail:\n#{$!}")
   exit(1)
 end
@@ -10,18 +10,7 @@ require 'jruby/jrubyc'
 require 'red_storm'
 require 'red_storm/application'
 
-DEP_STORM_VERSION = "0.8.1"
-DEP_JRUBY_VERSION = "1.6.8"
-INSTALL_IVY_VERSION = "2.2.0"
-
-DEFAULT_DEPENDENCIES = {
-  :storm_artifacts => [
-    "storm:storm:#{DEP_STORM_VERSION}, transitive=true",
-  ],
-  :topology_artifacts => [
-    "org.jruby:jruby-complete:#{DEP_JRUBY_VERSION}, transitive=false",
-  ],
-}
+INSTALL_IVY_VERSION = "2.3.0"
 
 task :launch, :env, :ruby_mode, :class_file do |t, args|
   # use ruby mode parameter or default to current interpreter version
@@ -32,7 +21,7 @@ task :launch, :env, :ruby_mode, :class_file do |t, args|
     RedStorm::Application.local_storm_command(args[:class_file], args[:ruby_mode])
   when "cluster"
     unless File.exist?(TARGET_CLUSTER_JAR)
-      puts("error: cluster jar file #{TARGET_CLUSTER_JAR} not found. Generate it usingw $redstorm jar DIR1 [DIR2, ...]")
+      puts("error: cluster jar file #{TARGET_CLUSTER_JAR} not found. Generate it using $redstorm jar DIR1 [DIR2, ...]")
       exit(1)
     end
     RedStorm::Application.cluster_storm_command(args[:class_file], args[:ruby_mode])
@@ -45,24 +34,24 @@ task :launch, :env, :ruby_mode, :class_file do |t, args|
 end
 
 task :clean do
-  ant.delete :dir => TARGET_DIR
+  ant.delete 'dir' => TARGET_DIR
 end
 
 task :clean_jar do
-  ant.delete :file => TARGET_CLUSTER_JAR
+  ant.delete 'file' => TARGET_CLUSTER_JAR
 end
 
 task :setup do
   puts("\n--> Setting up target directories")
-  ant.mkdir :dir => TARGET_DIR 
-  ant.mkdir :dir => TARGET_CLASSES_DIR 
-  ant.mkdir :dir => TARGET_DEPENDENCY_DIR
-  ant.mkdir :dir => TARGET_SRC_DIR
-  ant.mkdir :dir => TARGET_GEM_DIR
-  ant.mkdir :dir => TARGET_SPECS_DIR
-  ant.path :id => 'classpath' do  
-    fileset :dir => TARGET_DEPENDENCY_DIR  
-    fileset :dir => TARGET_CLASSES_DIR  
+  ant.mkdir 'dir' => TARGET_DIR 
+  ant.mkdir 'dir' => TARGET_CLASSES_DIR 
+  ant.mkdir 'dir' => TARGET_DEPENDENCY_DIR
+  ant.mkdir 'dir' => TARGET_SRC_DIR
+  ant.mkdir 'dir' => TARGET_GEM_DIR
+  ant.mkdir 'dir' => TARGET_SPECS_DIR
+  ant.path 'id' => 'classpath' do  
+    fileset 'dir' => TARGET_DEPENDENCY_DIR  
+    fileset 'dir' => TARGET_CLASSES_DIR  
   end  
 end
 
@@ -111,18 +100,23 @@ task :bundle, [:groups] => :setup do |t, args|
   defaulted_args = {:groups => 'default'}.merge(args.to_hash.delete_if{|k, v| v.to_s.empty?})
   groups = defaulted_args[:groups].split(':').map(&:to_sym)
   Bundler.definition.specs_for(groups).each do |spec|
-    unless spec.full_name =~ /^bundler-\d+/
-      destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"
-      unless File.directory?(destination_path)
-        puts("installing gem #{spec.full_name} into #{destination_path}")
-        # copy the actual gem dir
-        FileUtils.cp_r(spec.full_gem_path, destination_path)
-        # copy the gemspec into the specifications/ dir
-        FileUtils.cp_r(spec.loaded_from, TARGET_SPECS_DIR)
-        # strip the .git directory from git dependencies, it can be huge
-        FileUtils.rm_rf("#{destination_path}/.git")
-      end
-    end
+    next if spec.name == 'bundler'
+
+    # try to avoid infinite recursion
+    next if TARGET_GEM_DIR.start_with?(spec.full_gem_path)
+
+    destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"
+    next if File.directory?(destination_path)
+
+    puts("installing gem #{spec.full_name} into #{destination_path}")
+    # copy the actual gem dir
+    FileUtils.cp_r(spec.full_gem_path, destination_path)
+    # copy the evaluated gemspec into the specifications/ dir (we
+    # may not have enough info to reconstruct once we delete the
+    # .git directory)
+    File.open(File.join(TARGET_SPECS_DIR, File.basename(spec.loaded_from)), 'w'){|f| f.write(spec.to_ruby)}
+    # strip the .git directory from git dependencies, it can be huge
+    FileUtils.rm_rf("#{destination_path}/.git")
   end
 end
 
@@ -130,21 +124,21 @@ namespace :ivy do
   task :download do
     mkdir_p DST_IVY_DIR
     ant.get({
-      :src => "http://repo1.maven.org/maven2/org/apache/ivy/ivy/#{INSTALL_IVY_VERSION}/ivy-#{INSTALL_IVY_VERSION}.jar",
-      :dest => "#{DST_IVY_DIR}/ivy-#{INSTALL_IVY_VERSION}.jar",
-      :usetimestamp => true,
+      'src' => "http://repo1.maven.org/maven2/org/apache/ivy/ivy/#{INSTALL_IVY_VERSION}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      'dest' => "#{DST_IVY_DIR}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      'usetimestamp' => true,
     })
   end
 
   task :install => :download do
-    ant.path :id => 'ivy.lib.path' do
-      fileset :dir => DST_IVY_DIR, :includes => '*.jar'
+    ant.path 'id' => 'ivy.lib.path' do
+      fileset 'dir' => DST_IVY_DIR, 'includes' => '*.jar'
     end
 
     ant.taskdef({
-      :resource => "org/apache/ivy/ant/antlib.xml",
-      :classpathref => "ivy.lib.path",
-      #:uri => "antlib:org.apache.ivy.ant",
+      'resource' => "org/apache/ivy/ant/antlib.xml",
+      'classpathref' => "ivy.lib.path",
+      #'uri' => "antlib:org.apache.ivy.ant",
     })
   end
 end
@@ -152,37 +146,32 @@ end
 task :deps => "ivy:install" do
   puts("\n--> Installing dependencies")
 
-  dependencies = File.exists?(CUSTOM_DEPENDENCIES) ? eval(File.read(CUSTOM_DEPENDENCIES)) : DEFAULT_DEPENDENCIES
-  ant.configure :file => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
+  ant.configure 'file' => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
 
-  dependencies[:storm_artifacts].each do |dependency|
-    artifact, transitive = dependency.split(/\s*,\s*/)
-    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/storm", "default"]))
-  end
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_STORM_DEPENDENCIES) ? CUSTOM_IVY_STORM_DEPENDENCIES : DEFAULT_IVY_STORM_DEPENDENCIES 
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/storm/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"  
 
-  dependencies[:topology_artifacts].each do |dependency|
-    artifact, transitive = dependency.split(/\s*,\s*/)
-    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/topology", "default"]))
-  end
-end  
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_TOPOLOGY_DEPENDENCIES) ? CUSTOM_IVY_TOPOLOGY_DEPENDENCIES : DEFAULT_IVY_TOPOLOGY_DEPENDENCIES 
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/topology/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"
+end
 
 task :jar, [:include_dir] => [:clean_jar] do |t, args|
   puts("\n--> Generating JAR file #{TARGET_CLUSTER_JAR}")
 
-  ant.jar :destfile => TARGET_CLUSTER_JAR do
+  ant.jar 'destfile' => TARGET_CLUSTER_JAR do
     # rejar all topology jars
     Dir["target/dependency/topology/default/*.jar"].each do |jar|
       puts("Extracting #{jar}")
-      zipfileset :src => jar, :includes => "**/*"
+      zipfileset 'src' => jar, 'includes' => "**/*"
     end
-    fileset :dir => TARGET_DIR do
-      include :name => "gems/**"
+    fileset 'dir' => TARGET_DIR do
+      include 'name' => "gems/**"
     end
-    fileset :dir => TARGET_CLASSES_DIR
+    fileset 'dir' => TARGET_CLASSES_DIR
     # red_storm.rb and red_storm/* must be in root of jar so that "require 'red_storm'"
     # in bolts/spouts works in jar context
-    fileset :dir => TARGET_LIB_DIR do
-      exclude :name => "tasks/**"
+    fileset 'dir' => TARGET_LIB_DIR do
+      exclude 'name' => "tasks/**"
     end
     if args[:include_dir]
       dirs = args[:include_dir].split(":")
@@ -192,19 +181,19 @@ task :jar, [:include_dir] => [:clean_jar] do |t, args|
         resources_dirs = Dir.glob("#{dir}/**/resources")
         resources_dirs.each do |resources_dir|
           resources_parent = resources_dir.gsub("/resources", "")
-          fileset :dir => resources_parent do
-            include :name => "resources/**/*"
+          fileset 'dir' => resources_parent do
+            include 'name' => "resources/**/*"
           end
         end
       end
 
       # include complete source dir tree (note we don't care about potential duplicated resources dir)
-      fileset :dir => CWD do
-        dirs.each{|dir| include :name => "#{dir}/**/*"}
+      fileset 'dir' => CWD do
+        dirs.each{|dir| include 'name' => "#{dir}/**/*"}
       end
     end
     manifest do
-      attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
+      attribute 'name' => "Main-Class", 'value' => "redstorm.TopologyLauncher"
     end
   end
   puts("\nRedStorm generated JAR file #{TARGET_CLUSTER_JAR}")
@@ -213,15 +202,15 @@ end
 def build_java_dir(source_folder)
   puts("\n--> Compiling Java")
   ant.javac(
-    :srcdir => source_folder,
-    :destdir => TARGET_CLASSES_DIR,
-    :classpathref => 'classpath', 
-    :source => "1.6",
-    :target => "1.6",
-    :debug => "yes",
-    :includeantruntime => "no",
-    :verbose => false,
-    :listfiles => true
+    'srcdir' => source_folder,
+    'destdir' => TARGET_CLASSES_DIR,
+    'classpathref' => 'classpath', 
+    'source' => "1.7",
+    'target' => "1.7",
+    'debug' => "yes",
+    'includeantruntime' => "no",
+    'verbose' => false,
+    'listfiles' => true
   ) do
     # compilerarg :value => "-Xlint:unchecked"
   end 
@@ -242,30 +231,3 @@ def build_jruby(source_path)
     status =  JRuby::Compiler::compile_argv(argv)
   end
 end
-
-def truefalse(s)
-  return true if s.to_s.downcase =~ /1|yes|true/
-  return false if s.to_s.downcase =~ /0|no|false/
-  nil
-end
-
-def ivy_retrieve(org, mod, rev, transitive, dir, conf)
-  ant.resolve({
-    :organisation => org,
-    :module => mod,
-    :revision => rev,
-    :inline => true,
-    :transitive => truefalse(transitive),
-    :conf => conf,
-  })
-
-  ant.retrieve({
-    :organisation => org,
-    :module => mod,
-    :revision => rev,
-    :pattern => "#{dir}/[conf]/[artifact]-[revision].[ext]",
-    :inline => true,
-    :transitive => truefalse(transitive),
-    :conf => conf,
-  })
-end

data/redstorm.gemspec

@@ -0,0 +1,24 @@
+libdir = File.expand_path('../lib/', __FILE__)
+$:.unshift libdir unless $:.include?(libdir)
+
+require 'red_storm/version'
+
+Gem::Specification.new do |s|  
+  s.name        = 'redstorm'
+  s.version     = RedStorm::VERSION
+  s.authors     = ['Colin Surprenant']
+  s.email       = ['colin.surprenant@gmail.com']
+  s.homepage    = 'https://github.com/colinsurprenant/redstorm'
+  s.summary     = 'JRuby on Storm'
+  s.description = 'JRuby integration & DSL for the Storm distributed realtime computation system'
+ 
+  s.rubyforge_project = 'redstorm'
+  
+  s.files         = Dir.glob("{lib/**/*}") + Dir.glob("{ivy/*.xml}") + Dir.glob("{examples/**/*}") + Dir.glob("{src/**/*.java}") + Dir.glob("{bin/**/*}") + %w(redstorm.gemspec Rakefile README.md CHANGELOG.md LICENSE.md)
+  s.require_paths = ['lib']
+  s.bindir        = 'bin'
+  s.executables   = ['redstorm']
+
+  s.add_development_dependency 'rspec', '~> 2.11.0'
+  s.add_runtime_dependency 'rake'
+end

data/src/main/redstorm/storm/jruby/JRubyBatchBolt.java

@@ -0,0 +1,90 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.base.BaseBatchBolt;
+import backtype.storm.coordination.BatchOutputCollector;
+import backtype.storm.coordination.IBatchBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyBatchBolt extends BaseBatchBolt {
+  IBatchBolt _proxyBolt;
+  String _realBoltClassName;
+  String _baseClassPath; 
+  String[] _fields;
+
+  /**
+   * create a new JRubyBolt
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   */
+  public JRubyBatchBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realBoltClassName = realBoltClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void prepare(final Map stormConf, final TopologyContext context, final BatchOutputCollector collector, final Object id) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    _proxyBolt.prepare(stormConf, context, collector, id);
+  }
+
+  @Override
+  public void execute(Tuple input) {
+    _proxyBolt.execute(input);
+  }
+
+  @Override
+  public void finishBatch() {
+    _proxyBolt.finishBatch();
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IBatchBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+
+  private static IBatchBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.BatchBolt proxy = new redstorm.proxy.BatchBolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}