redstorm 0.4.0 → 0.5.0

data/Rakefile

@@ -1,14 +1,10 @@
-load 'lib/tasks/red_storm.rake'
+require 'rubygems'
+require 'rspec/core/rake_task'
 
-task :default => :spec
+load 'lib/tasks/red_storm.rake'
 
-begin
-  require 'rspec/core/rake_task'
-  desc "run specs"
-  task :spec do
-    system("ruby -v")
-    RSpec::Core::RakeTask.new
-  end
-rescue NameError, LoadError => e
-  puts e
+RSpec::Core::RakeTask.new(:spec) do
+  system("ruby -v")
 end
+
+task :default => :spec

data/TODO.md

@@ -1,4 +1 @@
 # TODO
-
-- expose the log4j logger in Simple{Topology|Bolt|Spout} (Java::org.apache.log4j.Logger.getLogger(...))
-- see if using bundler would provide better gems integration and easier jar packaging

data/examples/simple/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gem 'redis'

data/examples/simple/exclamation_topology.rb

@@ -7,21 +7,27 @@ require 'examples/simple/exclamation_bolt'
 module RedStorm
   module Examples
     class ExclamationTopology < RedStorm::SimpleTopology
-      spout TestWordSpout, :parallelism => 10
+      spout TestWordSpout, :parallelism => 5 do
+        debug true
+      end
       
-      bolt ExclamationBolt, :parallelism => 3 do
+      bolt ExclamationBolt, :parallelism => 2 do
         source TestWordSpout, :shuffle
+        # max_task_parallelism 1
       end
       
       bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
         source ExclamationBolt, :shuffle
+        # max_task_parallelism 1
+        debug true
       end
 
       configure do |env|
-        debug true
+        debug false
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
-          max_task_parallelism 3
+          max_task_parallelism 40
         when :cluster
           num_workers 20
           max_spout_pending(1000);

data/examples/simple/exclamation_topology2.rb

@@ -24,6 +24,7 @@ module RedStorm
 
       configure do |env|
         debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3

data/examples/simple/random_sentence_spout.rb

@@ -3,7 +3,6 @@ require 'red_storm'
 module RedStorm
   module Examples
     class RandomSentenceSpout < RedStorm::SimpleSpout
-      set :is_distributed => true
       output_fields :word
 
       on_send {@sentences[rand(@sentences.length)]}

data/examples/simple/redis_word_count_topology.rb

@@ -1,8 +1,9 @@
-require 'rubygems'
+require 'rubygems'      # required for remote cluster exec where TopolyLauncher + require rubygems is not called
+require 'red_storm'     # must be required before bundler for environment setup and after rubygems
+require 'bundler/setup'
+
 require 'redis'
 require 'thread'
-require 'red_storm'
-
 require 'examples/simple/word_count_bolt'
 
 module RedStorm
@@ -46,6 +47,7 @@ module RedStorm
 
       configure do |env|
         debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3

data/examples/simple/ruby_version_topology.rb

@@ -0,0 +1,32 @@
+require 'red_storm'
+
+# this example topology only prints the Ruby version string. No tuple is emitted.
+
+module RedStorm
+  module Examples
+    class VersionSpout < RedStorm::SimpleSpout
+      output_fields :dummy
+      on_init {log.info("****** JRuby version #{RUBY_VERSION}")}
+      on_send {}
+    end
+
+    class RubyVersionTopology < RedStorm::SimpleTopology
+      spout VersionSpout
+            
+      configure do |env|
+        debug true
+
+        # set the JRuby version property for this topology. this will only affect remote cluster execution
+        # for local execution use the --1.8|--1.9 switch when launching
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+      end
+
+      on_submit do |env|
+        if env == :local
+          sleep(1)
+          cluster.shutdown
+        end
+      end
+    end
+  end
+end

data/examples/simple/word_count_topology.rb

@@ -17,6 +17,7 @@ module RedStorm
 
       configure :word_count do |env|
         debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3

data/lib/red_storm.rb

@@ -1,10 +1,48 @@
+# we depends on rubygems begings loaded at this point for setting up gem/bundle environments
+# explicitely requiring rubygems is required in remote cluster environment
+require 'rubygems'
+
 module RedStorm
-  REDSTORM_HOME = File.expand_path(File.dirname(__FILE__) + '/..') unless defined?(REDSTORM_HOME)
+  LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
+  JAR_CONTEXT = !!(LAUNCH_PATH =~ /\.jar!$/)
+
+  if JAR_CONTEXT
+    REDSTORM_HOME = LAUNCH_PATH
+    TARGET_PATH = LAUNCH_PATH
+    BUNDLE_GEMFILE = "#{TARGET_PATH}/bundler/Gemfile"
+    BUNDLE_PATH = "#{TARGET_PATH}/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
+    GEM_PATH = "#{TARGET_PATH}/gems/"
+  else
+    REDSTORM_HOME = File.expand_path(LAUNCH_PATH + '/..')
+    TARGET_PATH = Dir.pwd
+    BUNDLE_GEMFILE = "#{TARGET_PATH}/target/gems/bundler/Gemfile"
+    BUNDLE_PATH = "#{TARGET_PATH}/target/gems/bundler/#{Gem.ruby_engine}/#{Gem::ConfigMap[:ruby_version]}/"
+    GEM_PATH = "#{TARGET_PATH}/target/gems/gems"
+  end
+
+  # setup bundler environment
+  ENV['BUNDLE_GEMFILE'] = RedStorm::BUNDLE_GEMFILE
+  ENV['BUNDLE_PATH'] = RedStorm::BUNDLE_PATH
+  ENV["GEM_PATH"] = RedStorm::GEM_PATH
+  ENV['BUNDLE_DISABLE_SHARED_GEMS'] = "1"
 end
 
+$:.unshift RedStorm::TARGET_PATH
+
+
 require 'red_storm/version'
 require 'red_storm/configuration'
 require 'red_storm/application'
 require 'red_storm/simple_bolt'
 require 'red_storm/simple_spout'
 require 'red_storm/simple_topology'
+
+# puts("************************ PWD=#{Dir.pwd}")
+# puts("************************ RedStorm::JAR_CONTEXT=#{RedStorm::JAR_CONTEXT}")
+# puts("************************ RedStorm::LAUNCH_PATH=#{RedStorm::LAUNCH_PATH}")
+# puts("************************ RedStorm::REDSTORM_HOME=#{RedStorm::REDSTORM_HOME}")
+# puts("************************ RedStorm::TARGET_PATH=#{RedStorm::TARGET_PATH}")
+# puts("************************ RedStorm::GEM_PATH=#{RedStorm::GEM_PATH}")
+# puts("************************ ENV['BUNDLE_GEMFILE']=#{ENV['BUNDLE_GEMFILE']}")
+# puts("************************ ENV['BUNDLE_PATH']=#{ENV['BUNDLE_PATH']}")
+# puts("************************ ENV['GEM_PATH']=#{ENV['GEM_PATH']}")

data/lib/red_storm/application.rb

@@ -1,27 +1,36 @@
 module RedStorm
+
+  DEFAULT_RUBY_VERSION = "--1.8"
+  RUNTIME = {}
   
   class Application 
     TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
 
     def usage
-      puts("usage: redstorm install|examples|jar <project_directory>|local <topology_class_file>")
+      puts("usage: redstorm [--1.8|--1.9]  install | deps | build | examples | gems [--gemfile GEMFILE_PATH] | jar DIR1, DIR2, ... | local TOPOLOGY_CLASS_PATH")
       exit(1)
     end
 
     def run(args)
       if args.size > 0
-        if ["install", "examples", "jar"].include?(args[0])
-          load(TASKS_FILE)
-          Rake::Task[args.shift].invoke(*args)
-        elsif args.size == 2 && ["local"].include?(args[0]) && File.exist?(args[1])
+        version = args.delete("--1.8") || args.delete("--1.9") || DEFAULT_RUBY_VERSION
+        RUNTIME['RUBY_VERSION'] = version
+
+        if ["install", "examples", "jar", "gems", "deps", "build"].include?(args[0])
           load(TASKS_FILE)
-          Rake::Task['launch'].invoke(*args)
-        else
-          usage
+          Rake::Task[args.shift].invoke(args.join(":"))
+          exit
+        elsif args.size >= 2 && args.include?("local") 
+          args.delete("local")
+          if args.size == 1
+            file = args[0]
+            load(TASKS_FILE)
+            Rake::Task['launch'].invoke("local", file)
+            exit
+          end
         end
-      else
-        usage
       end
+      usage
     end
   end
 end

data/lib/red_storm/configurator.rb

@@ -0,0 +1,25 @@
+module RedStorm
+
+  class Configurator
+    attr_reader :config
+
+    def initialize
+      @config = Backtype::Config.new
+    end
+
+    def set(attribute, value)
+      @config.put(attribute, value)
+    end
+
+    def method_missing(sym, *args)
+      config_method = "set#{self.class.camel_case(sym)}"
+      @config.send(config_method, *args)
+    end
+
+    private
+
+    def self.camel_case(s)
+      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+  end
+end

data/lib/red_storm/loggable.rb

@@ -0,0 +1,13 @@
+module RedStorm
+  module Loggable
+
+    def self.log
+      @log ||= Logger.getLogger(self.name)
+    end
+
+    def log
+      self.class.log
+    end
+
+  end
+end

data/lib/red_storm/proxy/bolt.rb

@@ -8,6 +8,10 @@ java_import 'backtype.storm.tuple.Tuple'
 java_import 'backtype.storm.tuple.Fields'
 java_import 'backtype.storm.tuple.Values'
 java_import 'java.util.Map'
+java_import 'org.apache.log4j.Logger'
+module Backtype
+  java_import 'backtype.storm.Config'
+end
 
 java_package 'redstorm.proxy'
 
@@ -52,4 +56,9 @@ class Bolt
   def declareOutputFields(declarer)
     @real_bolt.declare_output_fields(declarer)
   end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
 end

data/lib/red_storm/proxy/spout.rb

@@ -8,6 +8,10 @@ java_import 'backtype.storm.tuple.Tuple'
 java_import 'backtype.storm.tuple.Fields'
 java_import 'backtype.storm.tuple.Values'
 java_import 'java.util.Map'
+java_import 'org.apache.log4j.Logger'
+module Backtype
+  java_import 'backtype.storm.Config'
+end
 
 java_package 'redstorm.proxy'
 
@@ -17,7 +21,6 @@ java_package 'redstorm.proxy'
 # The real spout class implementation must define these methods:
 # - open(conf, context, collector)
 # - next_tuple
-# - is_distributed
 # - declare_output_fields
 #
 # and optionnaly:
@@ -25,6 +28,7 @@ java_package 'redstorm.proxy'
 # - fail(msg_id)
 # - close
 #
+
 class Spout
   java_implements IRichSpout
 
@@ -36,11 +40,6 @@ class Spout
     @real_spout = Object.module_eval(real_spout_class_name).new
   end
 
-  java_signature 'boolean isDistributed()'
-  def isDistributed
-    @real_spout.respond_to?(:is_distributed) ? @real_spout.is_distributed : false
-  end
-
   java_signature 'void open(Map, TopologyContext, SpoutOutputCollector)'
   def open(conf, context, collector)
     @real_spout.open(conf, context, collector)
@@ -51,6 +50,16 @@ class Spout
     @real_spout.close if @real_spout.respond_to?(:close)
   end
 
+  java_signature 'void activate()'
+  def activate
+    @real_spout.activate if @real_spout.respond_to?(:activate)
+  end
+
+  java_signature 'void deactivate()'
+  def deactivate
+    @real_spout.deactivate if @real_spout.respond_to?(:deactivate)
+  end
+
   java_signature 'void nextTuple()'
   def nextTuple
     @real_spout.next_tuple
@@ -70,4 +79,10 @@ class Spout
   def declareOutputFields(declarer)
     @real_spout.declare_output_fields(declarer)
   end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
 end

data/lib/red_storm/simple_bolt.rb

@@ -1,3 +1,5 @@
+require 'red_storm/configurator'
+
 module RedStorm
 
   class SimpleBolt
@@ -5,10 +7,18 @@ module RedStorm
 
     # DSL class methods
 
+    def self.log
+      @log ||= Logger.getLogger(self.name)
+    end
+
     def self.output_fields(*fields)
       @fields = fields.map(&:to_s)
     end
 
+    def self.configure(&configure_block)
+      @configure_block = block_given? ? configure_block : lambda {}
+    end
+
     def self.on_receive(*args, &on_receive_block)
       options = args.last.is_a?(Hash) ? args.pop : {}
       method_name = args.first
@@ -27,6 +37,10 @@ module RedStorm
 
     # DSL instance methods
 
+    def log
+      self.class.log
+    end
+
     def unanchored_emit(*values)
       @collector.emit(Values.new(*values)) 
     end
@@ -64,17 +78,26 @@ module RedStorm
       declarer.declare(Fields.new(self.class.fields))
     end
 
-    # default optional dsl methods/callbacks
+    def get_component_configuration
+      configurator = Configurator.new
+      configurator.instance_exec(&self.class.configure_block)
+      configurator.config
+    end
+
+    private
 
+    # default noop optional dsl callbacks
     def on_init; end
     def on_close; end
 
-    private
-
     def self.fields
       @fields ||= []
     end
 
+    def self.configure_block
+      @configure_block ||= lambda {}
+    end
+
     def self.on_receive_block
       @on_receive_block ||= lambda {|tuple| self.send(:on_receive, tuple)}
     end

data/lib/red_storm/simple_spout.rb

@@ -1,3 +1,5 @@
+require 'red_storm/configurator'
+
 module RedStorm
 
   class SimpleSpout
@@ -5,8 +7,12 @@ module RedStorm
 
     # DSL class methods
 
-    def self.set(options = {})
-      self.spout_options.merge!(options)
+    def self.configure(&configure_block)
+      @configure_block = block_given? ? configure_block : lambda {}
+    end
+
+    def self.log
+      @log ||= Logger.getLogger(self.name)
     end
 
     def self.output_fields(*fields)
@@ -29,6 +35,14 @@ module RedStorm
       @on_close_block = block_given? ? on_close_block : lambda {self.send(method_name || :on_close)}
     end
 
+    def self.on_activate(method_name = nil, &on_activate_block)
+      @on_activate_block = block_given? ? on_activate_block : lambda {self.send(method_name || :on_activate)}
+    end
+
+    def self.on_deactivate(method_name = nil, &on_deactivate_block)
+      @on_deactivate_block = block_given? ? on_deactivate_block : lambda {self.send(method_name || :on_deactivate)}
+    end
+
     def self.on_ack(method_name = nil, &on_ack_block)
       @on_ack_block = block_given? ? on_ack_block : lambda {|msg_id| self.send(method_name || :on_ack, msg_id)}
     end
@@ -43,6 +57,10 @@ module RedStorm
       @collector.emit(Values.new(*values)) 
     end
 
+    def log
+      self.class.log
+    end
+
     # Spout proxy interface
 
     def next_tuple
@@ -68,12 +86,16 @@ module RedStorm
       instance_exec(&self.class.on_close_block)
     end
 
-    def declare_output_fields(declarer)
-      declarer.declare(Fields.new(self.class.fields))
+    def activate
+      instance_exec(&self.class.on_activate_block)
     end
 
-    def is_distributed
-      self.class.is_distributed?
+    def deactivate
+      instance_exec(&self.class.on_deactivate_block)
+    end
+
+    def declare_output_fields(declarer)
+      declarer.declare(Fields.new(self.class.fields))
     end
 
     def ack(msg_id)
@@ -84,19 +106,30 @@ module RedStorm
       instance_exec(msg_id, &self.class.on_fail_block)
     end
 
-    # default optional dsl methods/callbacks
+    def get_component_configuration
+      configurator = Configurator.new
+      configurator.instance_exec(&self.class.configure_block)
+      configurator.config
+    end
+
+    private
 
+    # default optional noop dsl methods/callbacks
     def on_init; end
     def on_close; end
+    def on_activate; end
+    def on_deactivate; end
     def on_ack(msg_id); end
     def on_fail(msg_id); end
 
-    private
-
     def self.fields
       @fields ||= []
     end
 
+    def self.configure_block
+      @configure_block ||= lambda {}
+    end
+
     def self.on_send_block
       @on_send_block ||= lambda {self.send(:on_send)}
     end
@@ -109,6 +142,14 @@ module RedStorm
       @on_close_block ||= lambda {self.send(:on_close)}
     end
 
+    def self.on_activate_block
+      @on_activate_block ||= lambda {self.send(:on_activate)}
+    end
+
+    def self.on_deactivate_block
+      @on_deactivate_block ||= lambda {self.send(:on_deactivate)}
+    end
+
     def self.on_ack_block
       @on_ack_block ||= lambda {|msg_id| self.send(:on_ack, msg_id)}
     end
@@ -121,14 +162,6 @@ module RedStorm
       @send_options ||= {:emit => true}
     end
 
-    def self.spout_options
-      @spout_options ||= {:is_distributed => false}
-    end
-
-    def self.is_distributed?
-      !!self.spout_options[:is_distributed]
-    end
-
     def self.emit?
       !!self.send_options[:emit]
     end