kb-redstorm 0.6.4

data/lib/red_storm/topology_launcher.rb

@@ -0,0 +1,54 @@
+require 'java'
+
+# see https://github.com/colinsurprenant/redstorm/issues/7
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_import 'backtype.storm.LocalCluster'
+java_import 'backtype.storm.LocalDRPC'
+java_import 'backtype.storm.StormSubmitter'
+java_import 'backtype.storm.topology.TopologyBuilder'
+java_import 'backtype.storm.drpc.LinearDRPCTopologyBuilder'
+java_import 'backtype.storm.tuple.Fields'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'backtype.storm.tuple.Values'
+
+java_import 'redstorm.storm.jruby.JRubyBolt'
+java_import 'redstorm.storm.jruby.JRubySpout'
+java_import 'redstorm.storm.jruby.JRubyBatchSpout'
+
+java_package 'redstorm'
+
+# TopologyLauncher is the application entry point when launching a topology. Basically it will
+# call require on the specified Ruby topology class file path and call its start method
+class TopologyLauncher
+
+  java_signature 'void main(String[])'
+  def self.main(args)
+    unless args.size > 1
+      puts("Usage: redstorm local|cluster topology_class_file_name")
+      exit(1)
+    end
+
+    env = args[0].to_sym
+    class_path = args[1]
+
+    launch_path = Dir.pwd
+    $:.unshift File.expand_path(launch_path)
+    $:.unshift File.expand_path(launch_path + '/lib')
+    $:.unshift File.expand_path(launch_path + '/target/lib')
+
+    require "#{class_path}"
+
+    topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
+    puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
+    RedStorm::Configuration.topology_class.new.start(class_path, env)
+  end
+
+  private
+
+  def self.camel_case(s)
+    s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+end

data/lib/red_storm/version.rb

@@ -0,0 +1,3 @@
+module RedStorm
+  VERSION = '0.6.4'
+end

data/lib/tasks/red_storm.rake

@@ -0,0 +1,272 @@
+begin
+  require 'ant'
+rescue
+  puts("error: unable to load Ant, make sure Ant is installed, in your PATH")
+  puts("       and $ANT_HOME is defined properly.")
+  puts("\nerror detail:\n#{$!}")
+  exit(1)
+end
+
+require 'jruby/jrubyc'
+require 'red_storm'
+require 'red_storm/application'
+
+DEP_STORM_VERSION = "0.8.2"
+DEP_JRUBY_VERSION = "1.6.8"
+INSTALL_IVY_VERSION = "2.2.0"
+
+DEFAULT_DEPENDENCIES = {
+  :storm_artifacts => [
+    "storm:storm:#{DEP_STORM_VERSION}, transitive=true",
+  ],
+  :topology_artifacts => [
+    "org.jruby:jruby-complete:#{DEP_JRUBY_VERSION}, transitive=false",
+  ],
+}
+
+task :launch, :env, :ruby_mode, :class_file do |t, args|
+  # use ruby mode parameter or default to current interpreter version
+  version_token = RedStorm.jruby_mode_token(args[:ruby_mode])
+
+  command = case args[:env]
+  when "local"
+    RedStorm::Application.local_storm_command(args[:class_file], args[:ruby_mode])
+  when "cluster"
+    unless File.exist?(TARGET_CLUSTER_JAR)
+      puts("error: cluster jar file #{TARGET_CLUSTER_JAR} not found. Generate it usingw $redstorm jar DIR1 [DIR2, ...]")
+      exit(1)
+    end
+    RedStorm::Application.cluster_storm_command(args[:class_file], args[:ruby_mode])
+  end
+
+  puts("launching #{command}")
+  unless system(command)
+    puts($!)
+  end
+end
+
+task :clean do
+  ant.delete :dir => TARGET_DIR
+end
+
+task :clean_jar do
+  ant.delete :file => TARGET_CLUSTER_JAR
+end
+
+task :setup do
+  puts("\n--> Setting up target directories")
+  ant.mkdir :dir => TARGET_DIR
+  ant.mkdir :dir => TARGET_CLASSES_DIR
+  ant.mkdir :dir => TARGET_DEPENDENCY_DIR
+  ant.mkdir :dir => TARGET_SRC_DIR
+  ant.mkdir :dir => TARGET_GEM_DIR
+  ant.mkdir :dir => TARGET_SPECS_DIR
+  ant.path :id => 'classpath' do
+    fileset :dir => TARGET_DEPENDENCY_DIR
+    fileset :dir => TARGET_CLASSES_DIR
+  end
+end
+
+task :install => [:deps, :build] do
+  puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}")
+end
+
+task :examples do
+  if File.identical?(SRC_EXAMPLES, DST_EXAMPLES)
+    STDERR.puts("error: cannot copy examples into itself")
+    exit(1)
+  end
+  if File.exist?(DST_EXAMPLES)
+    STDERR.puts("error: directory #{DST_EXAMPLES} already exists")
+    exit(1)
+  end
+
+  puts("\n--> Installing examples into #{DST_EXAMPLES}")
+  FileUtils.mkdir(DST_EXAMPLES)
+  FileUtils.cp_r(Dir["#{SRC_EXAMPLES}/*"], DST_EXAMPLES)
+end
+
+task :copy_red_storm do
+  FileUtils.cp_r(REDSTORM_LIB_DIR, TARGET_DIR)
+end
+
+task :build => [:setup, :copy_red_storm] do
+  # compile the JRuby proxy classes to Java
+  build_jruby("#{REDSTORM_LIB_DIR}/red_storm/proxy")
+
+  # compile the generated Java proxy classes
+  build_java_dir("#{TARGET_SRC_DIR}")
+
+  # generate the JRuby topology launcher
+  build_jruby("#{REDSTORM_LIB_DIR}/red_storm/topology_launcher.rb")
+
+  # compile the JRuby proxy classes
+  build_java_dir("#{REDSTORM_JAVA_SRC_DIR}")
+
+  # compile the JRuby proxy classes
+  build_java_dir("#{TARGET_SRC_DIR}")
+end
+
+task :bundle, [:groups] => :setup do |t, args|
+  require 'bundler'
+  defaulted_args = {:groups => 'default'}.merge(args.to_hash.delete_if{|k, v| v.to_s.empty?})
+  groups = defaulted_args[:groups].split(':').map(&:to_sym)
+  Bundler.definition.specs_for(groups).each do |spec|
+    unless spec.full_name =~ /^bundler-\d+/
+      destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"
+      unless File.directory?(destination_path)
+        puts("installing gem #{spec.full_name} into #{destination_path}")
+        # copy the actual gem dir
+        FileUtils.cp_r(spec.full_gem_path, destination_path)
+        # copy the gemspec into the specifications/ dir
+        FileUtils.cp_r(spec.loaded_from, TARGET_SPECS_DIR)
+        # strip the .git directory from git dependencies, it can be huge
+        FileUtils.rm_rf("#{destination_path}/.git")
+      end
+    end
+  end
+end
+
+namespace :ivy do
+  task :download do
+    mkdir_p DST_IVY_DIR
+    ant.get({
+      :src => "http://repo1.maven.org/maven2/org/apache/ivy/ivy/#{INSTALL_IVY_VERSION}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      :dest => "#{DST_IVY_DIR}/ivy-#{INSTALL_IVY_VERSION}.jar",
+      :usetimestamp => true,
+    })
+  end
+
+  task :install => :download do
+    ant.path :id => 'ivy.lib.path' do
+      fileset :dir => DST_IVY_DIR, :includes => '*.jar'
+    end
+
+    ant.taskdef({
+      :resource => "org/apache/ivy/ant/antlib.xml",
+      :classpathref => "ivy.lib.path",
+      #:uri => "antlib:org.apache.ivy.ant",
+    })
+  end
+end
+
+task :deps => "ivy:install" do
+  puts("\n--> Installing dependencies")
+
+  dependencies = File.exists?(CUSTOM_DEPENDENCIES) ? eval(File.read(CUSTOM_DEPENDENCIES)) : DEFAULT_DEPENDENCIES
+  ant.configure :file => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
+
+  dependencies[:storm_artifacts].each do |dependency|
+    artifact, transitive = dependency.split(/\s*,\s*/)
+    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/storm", "default"]))
+  end
+
+  dependencies[:topology_artifacts].each do |dependency|
+    artifact, transitive = dependency.split(/\s*,\s*/)
+    ivy_retrieve(*artifact.split(':').concat([transitive.split(/\s*=\s*/).last, "#{TARGET_DEPENDENCY_DIR}/topology", "default"]))
+  end
+end
+
+task :jar, [:include_dir] => [:clean_jar] do |t, args|
+  puts("\n--> Generating JAR file #{TARGET_CLUSTER_JAR}")
+
+  ant.jar :destfile => TARGET_CLUSTER_JAR do
+    # rejar all topology jars
+    Dir["target/dependency/topology/default/*.jar"].each do |jar|
+      puts("Extracting #{jar}")
+      zipfileset :src => jar, :includes => "**/*"
+    end
+    fileset :dir => TARGET_DIR do
+      include :name => "gems/**"
+    end
+    fileset :dir => TARGET_CLASSES_DIR
+    # red_storm.rb and red_storm/* must be in root of jar so that "require 'red_storm'"
+    # in bolts/spouts works in jar context
+    fileset :dir => TARGET_LIB_DIR do
+      exclude :name => "tasks/**"
+    end
+    if args[:include_dir]
+      dirs = args[:include_dir].split(":")
+
+      # first add any resources/ dir in the tree in the jar root - requirement for ShellBolt multilang resources
+      dirs.each do |dir|
+        resources_dirs = Dir.glob("#{dir}/**/resources")
+        resources_dirs.each do |resources_dir|
+          resources_parent = resources_dir.gsub("/resources", "")
+          fileset :dir => resources_parent do
+            include :name => "resources/**/*"
+          end
+        end
+      end
+
+      # include complete source dir tree (note we don't care about potential duplicated resources dir)
+      fileset :dir => CWD do
+        dirs.each{|dir| include :name => "#{dir}/**/*"}
+      end
+    end
+    manifest do
+      attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
+    end
+  end
+  puts("\nRedStorm generated JAR file #{TARGET_CLUSTER_JAR}")
+end
+
+def build_java_dir(source_folder)
+  puts("\n--> Compiling Java")
+  ant.javac(
+    :srcdir => source_folder,
+    :destdir => TARGET_CLASSES_DIR,
+    :classpathref => 'classpath',
+    :source => "1.6",
+    :target => "1.6",
+    :debug => "yes",
+    :includeantruntime => "no",
+    :verbose => false,
+    :listfiles => true
+  ) do
+    # compilerarg :value => "-Xlint:unchecked"
+  end
+end
+
+def build_jruby(source_path)
+  puts("\n--> Compiling JRuby")
+  Dir.chdir(RedStorm::REDSTORM_HOME) do
+    argv = []
+    argv << '-t' << TARGET_SRC_DIR
+    argv << '--verbose'
+    argv << '--java'
+    Dir["#{TARGET_DEPENDENCY_DIR}/storm/default/*.jar"].each do |jar|
+      argv << '-c' << %("#{jar}")
+    end
+    argv << '-c' << %("#{TARGET_CLASSES_DIR}")
+    argv << source_path
+    status =  JRuby::Compiler::compile_argv(argv)
+  end
+end
+
+def truefalse(s)
+  return true if s.to_s.downcase =~ /1|yes|true/
+  return false if s.to_s.downcase =~ /0|no|false/
+  nil
+end
+
+def ivy_retrieve(org, mod, rev, transitive, dir, conf)
+  ant.resolve({
+    :organisation => org,
+    :module => mod,
+    :revision => rev,
+    :inline => true,
+    :transitive => truefalse(transitive),
+    :conf => conf,
+  })
+
+  ant.retrieve({
+    :organisation => org,
+    :module => mod,
+    :revision => rev,
+    :pattern => "#{dir}/[conf]/[artifact]-[revision].[ext]",
+    :inline => true,
+    :transitive => truefalse(transitive),
+    :conf => conf,
+  })
+end

data/src/main/redstorm/storm/jruby/JRubyBatchSpout.java

@@ -0,0 +1,89 @@
+package redstorm.storm.jruby;
+
+import storm.trident.operation.TridentCollector;
+import backtype.storm.task.TopologyContext;
+import storm.trident.spout.IBatchSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBatchSpout class is a simple proxy class to the actual spout implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the spout to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class).
+ *
+ * Note that the JRuby spout proxy class is instanciated in the open method which is called after
+ * deserialization at the worker and in both the declareOutputFields and isDistributed methods which
+ * are called once before serialization at topology creation.
+ */
+public class JRubyBatchSpout implements IBatchSpout {
+  IBatchSpout _proxySpout;
+  String _realSpoutClassName;
+  String _baseClassPath;
+  String[] _fields;
+
+  /**
+   * create a new JRubyBatchSpout
+   *
+   * @param baseClassPath the topology/project base JRuby class file path
+   * @param realSpoutClassName the fully qualified JRuby spout implementation class name
+   */
+  public JRubyBatchSpout(String baseClassPath, String realSpoutClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void open(final Map conf, final TopologyContext context) {
+    // create instance of the jruby proxy class here, after deserialization in the workers.
+    _proxySpout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    _proxySpout.open(conf, context);
+  }
+
+  @Override
+  public void emitBatch(final long batchId, final TridentCollector collector) {
+    _proxySpout.emitBatch(batchId, collector);
+  }
+
+
+  @Override
+  public void close() {
+    _proxySpout.close();
+  }
+
+  @Override
+  public void ack(final long batchId) {
+    _proxySpout.ack(batchId);
+  }
+
+  @Override
+  public Fields getOutputFields() {
+    // getOutputFields is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getOutputFields();
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    IBatchSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+
+  private static IBatchSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
+    try {
+      redstorm.proxy.BatchSpout proxy = new redstorm.proxy.BatchSpout(baseClassPath, realSpoutClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyBolt.java

@@ -0,0 +1,88 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+/**
+ * the JRubyBolt class is a simple proxy class to the actual bolt implementation in JRuby.
+ * this proxy is required to bypass the serialization/deserialization process when dispatching
+ * the bolts to the workers. JRuby does not yet support serialization from Java
+ * (Java serialization call on a JRuby class). 
+ *
+ * Note that the JRuby bolt proxy class is instanciated in the prepare method which is called after 
+ * deserialization at the worker and in the declareOutputFields method which is called once before 
+ * serialization at topology creation. 
+ */
+public class JRubyBolt implements IRichBolt {
+  IRichBolt _proxyBolt;
+  String _realBoltClassName;
+  String _baseClassPath;
+  String[] _fields;
+
+  /**
+   * create a new JRubyBolt
+   * 
+   * @param baseClassPath the topology/project base JRuby class file path 
+   * @param realBoltClassName the fully qualified JRuby bolt implementation class name
+   */
+  public JRubyBolt(String baseClassPath, String realBoltClassName, String[] fields) {
+    _baseClassPath = baseClassPath;
+    _realBoltClassName = realBoltClassName;
+    _fields = fields;
+  }
+
+  @Override
+  public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
+    // create instance of the jruby class here, after deserialization in the workers.
+    _proxyBolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    _proxyBolt.prepare(stormConf, context, collector);
+  }
+
+  @Override
+  public void execute(Tuple input) {
+    _proxyBolt.execute(input);
+  }
+
+  @Override
+  public void cleanup() {
+    _proxyBolt.cleanup();
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    // declareOutputFields is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+
+  private static IRichBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
+    try {
+      redstorm.proxy.Bolt proxy = new redstorm.proxy.Bolt(baseClassPath, realBoltClassName);
+      return proxy;
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}