kb-redstorm 0.6.4

data/lib/red_storm/configuration.rb

@@ -0,0 +1,16 @@
+module RedStorm
+  module Configuration
+    extend self
+
+    @topology_class = nil
+
+    def topology_class=(clazz)
+      @topology_class = clazz
+    end
+
+    def topology_class
+      @topology_class
+    end
+
+  end
+end

data/lib/red_storm/configurator.rb

@@ -0,0 +1,26 @@
+module RedStorm
+
+  class Configurator
+    attr_reader :config
+
+    def initialize(defaults = {})
+      @config = Backtype::Config.new
+      defaults.each{|attribute, value| @config.put(attribute.to_s, value)}
+    end
+
+    def set(attribute, value)
+      @config.put(attribute.to_s, value)
+    end
+
+    def method_missing(sym, *args)
+      config_method = "set#{self.class.camel_case(sym)}"
+      @config.send(config_method, *args)
+    end
+
+    private
+
+    def self.camel_case(s)
+      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+  end
+end

data/lib/red_storm/environment.rb

@@ -0,0 +1,41 @@
+module RedStorm
+
+  LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
+  JAR_CONTEXT = !!(LAUNCH_PATH =~ /\.jar!\/red_storm$/)
+
+  if JAR_CONTEXT
+    BASE_PATH = File.expand_path(LAUNCH_PATH + '/..')
+    REDSTORM_HOME = BASE_PATH
+    TARGET_PATH = BASE_PATH
+  else
+    BASE_PATH = Dir.pwd
+    REDSTORM_HOME = File.expand_path(LAUNCH_PATH + '/../..')
+    TARGET_PATH = "#{BASE_PATH}/target"
+  end
+  
+  unless defined?(SPECS_CONTEXT)
+    GEM_PATH = "#{TARGET_PATH}/gems/"
+    ENV["GEM_PATH"] = GEM_PATH
+    ENV["GEM_HOME"] = GEM_PATH
+  end
+
+  def current_ruby_mode
+    RUBY_VERSION =~ /(\d+\.\d+)(\.\d+)*/
+    raise("unknown Ruby version #{$1}") unless $1 == "1.8" || $1 == "1.9"
+    $1
+  end
+
+  def jruby_mode_token(ruby_version = nil)
+    version_map = {"1.8" => "RUBY1_8", "--1.8" => "RUBY1_8", "1.9" => "RUBY1_9", "--1.9" => "RUBY1_9"}
+    version_map[ruby_version.to_s] || version_map[RedStorm.current_ruby_mode]
+  end
+
+  module_function :current_ruby_mode, :jruby_mode_token
+
+  # puts("*** LAUNCH_PATH=#{LAUNCH_PATH}")
+  # puts("*** JAR_CONTEXT=#{JAR_CONTEXT}")
+  # puts("*** BASE_PATH=#{BASE_PATH}")
+  # puts("*** REDSTORM_HOME=#{REDSTORM_HOME}")
+  # puts("*** TARGET_PATH=#{TARGET_PATH}")
+  # puts("*** GEM_PATH=#{GEM_PATH}") if defined?(GEM_PATH)
+end

data/lib/red_storm/loggable.rb

@@ -0,0 +1,15 @@
+require 'java'
+
+module RedStorm
+  module Loggable
+
+    def self.log
+      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+    end
+
+    def log
+      self.class.log
+    end
+
+  end
+end

data/lib/red_storm/proxy/batch_spout.rb

@@ -0,0 +1,71 @@
+require 'java'
+
+java_import 'storm.trident.operation.TridentCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'storm.trident.spout.IBatchSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'backtype.storm.tuple.Fields'
+java_import 'backtype.storm.tuple.Values'
+java_import 'java.util.Map'
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the BatchSpout class is a proxy to the real batch spout to avoid having to deal with all the
+# Java artifacts when creating a spout.
+#
+# The real batch spout class implementation must define these methods:
+# - open(conf, context, collector)
+# - emitBatch
+# - getOutputFields
+# - ack(batch_id)
+#
+# and optionnaly:
+# - close
+#
+
+class BatchSpout
+  java_implements IBatchSpout
+
+  java_signature 'IBatchSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'void open(Map, TopologyContext)'
+  def open(conf, context)
+    @real_spout.open(conf, context)
+  end
+
+  java_signature 'void close()'
+  def close
+    @real_spout.close if @real_spout.respond_to?(:close)
+  end
+
+  java_signature 'void emitBatch(long, TridentCollector)'
+  def emitBatch(batch_id, collector)
+    @real_spout.emit_batch(batch_id, collector)
+  end
+
+  java_signature 'void ack(long)'
+  def ack(batch_id)
+    @real_spout.ack(batch_id)
+  end
+
+  java_signature 'Fields getOutputFields()'
+  def getOutputFields
+    @real_spout.get_output_fields()
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/proxy/bolt.rb

@@ -0,0 +1,63 @@
+require 'java'
+
+java_import 'backtype.storm.task.OutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.topology.IRichBolt'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'backtype.storm.tuple.Fields'
+java_import 'backtype.storm.tuple.Values'
+java_import 'java.util.Map'
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the Bolt class is a proxy to the real bolt to avoid having to deal with all the
+# Java artifacts when creating a bolt.
+#
+# The real bolt class implementation must define these methods:
+# - prepare(conf, context, collector)
+# - execute(tuple)
+# - declare_output_fields
+#
+# and optionnaly:
+# - cleanup
+#
+class Bolt
+  java_implements IRichBolt
+
+  java_signature 'IRichBolt (String base_class_path, String real_bolt_class_name)'
+  def initialize(base_class_path, real_bolt_class_name)
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  end
+
+  java_signature 'void prepare(Map, TopologyContext, OutputCollector)'
+  def prepare(conf, context, collector)
+    @real_bolt.prepare(conf, context, collector)
+  end
+
+  java_signature 'void execute(Tuple)'
+  def execute(tuple)
+    @real_bolt.execute(tuple)
+  end
+
+  java_signature 'void cleanup()'
+  def cleanup
+    @real_bolt.cleanup if @real_bolt.respond_to?(:cleanup)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_bolt.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
+end

data/lib/red_storm/proxy/proxy_function.rb

@@ -0,0 +1,48 @@
+require 'java'
+
+
+java_import 'storm.trident.tuple.TridentTuple'
+
+java_import 'storm.trident.operation.TridentCollector'
+
+java_import 'java.util.Map'
+
+java_import 'storm.trident.operation.TridentOperationContext'
+
+java_import 'storm.trident.operation.Function'
+
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+class ProxyFunction
+  java_implements Function
+
+  java_signature 'Function (String base_class_path, String real_class_name)'
+  def initialize(base_class_path, real_class_name)
+    @real = Object.module_eval(real_class_name).new
+  rescue NameError
+    require base_class_path
+    @real = Object.module_eval(real_class_name).new
+  end
+
+  java_signature 'void execute(TridentTuple, TridentCollector)'
+  def execute(_trident_tuple, _trident_collector)
+    @real.execute(_trident_tuple, _trident_collector)
+  end
+
+  java_signature 'void cleanup()'
+  def cleanup()
+    @real.cleanup()
+  end
+
+  java_signature 'void prepare(Map, TridentOperationContext)'
+  def prepare(_map, _trident_operation_context)
+    @real.prepare(_map, _trident_operation_context)
+  end
+
+
+end

data/lib/red_storm/proxy/spout.rb

@@ -0,0 +1,87 @@
+require 'java'
+
+java_import 'backtype.storm.spout.SpoutOutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.topology.IRichSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'backtype.storm.tuple.Fields'
+java_import 'backtype.storm.tuple.Values'
+java_import 'java.util.Map'
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the Spout class is a proxy to the real spout to avoid having to deal with all the
+# Java artifacts when creating a spout.
+#
+# The real spout class implementation must define these methods:
+# - open(conf, context, collector)
+# - next_tuple
+# - declare_output_fields
+#
+# and optionnaly:
+# - ack(msg_id)
+# - fail(msg_id)
+# - close
+#
+
+class Spout
+  java_implements IRichSpout
+
+  java_signature 'IRichSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'void open(Map, TopologyContext, SpoutOutputCollector)'
+  def open(conf, context, collector)
+    @real_spout.open(conf, context, collector)
+  end
+
+  java_signature 'void close()'
+  def close
+    @real_spout.close if @real_spout.respond_to?(:close)
+  end
+
+  java_signature 'void activate()'
+  def activate
+    @real_spout.activate if @real_spout.respond_to?(:activate)
+  end
+
+  java_signature 'void deactivate()'
+  def deactivate
+    @real_spout.deactivate if @real_spout.respond_to?(:deactivate)
+  end
+
+  java_signature 'void nextTuple()'
+  def nextTuple
+    @real_spout.next_tuple
+  end
+
+  java_signature 'void ack(Object)'
+  def ack(msg_id)
+    @real_spout.ack(msg_id) if @real_spout.respond_to?(:ack)
+  end
+
+  java_signature 'void fail(Object)'
+  def fail(msg_id)
+    @real_spout.fail(msg_id) if @real_spout.respond_to?(:fail)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_spout.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/simple_bolt.rb

@@ -0,0 +1,135 @@
+require 'java'
+require 'red_storm/configurator'
+
+module RedStorm
+
+  class SimpleBolt
+    attr_reader :collector, :context, :config
+
+    # DSL class methods
+
+    def self.log
+      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+    end
+
+    def self.output_fields(*fields)
+      @fields = fields.map(&:to_s)
+    end
+
+    def self.configure(&configure_block)
+      @configure_block = block_given? ? configure_block : lambda {}
+    end
+
+    def self.on_receive(*args, &on_receive_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      method_name = args.first
+
+      self.receive_options.merge!(options)
+      @on_receive_block = block_given? ? on_receive_block : lambda {|tuple| self.send(method_name || :on_receive, tuple)}
+    end
+
+    def self.on_init(method_name = nil, &on_init_block)
+      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
+    end
+
+    def self.on_close(method_name = nil, &close_block)
+      @close_block = block_given? ? close_block : lambda {self.send(method_name || :on_close)}
+    end
+
+    # DSL instance methods
+
+    def log
+      self.class.log
+    end
+
+    def unanchored_emit(*values)
+      @collector.emit(Values.new(*values)) 
+    end
+
+    def anchored_emit(tuple, *values)
+      @collector.emit(tuple, Values.new(*values)) 
+    end
+
+    def ack(tuple)
+      @collector.ack(tuple)
+    end
+
+    def fail(tuple)
+      @collector.fail(tuple)
+    end
+
+    # Bolt proxy interface
+
+    def execute(tuple)
+      output = instance_exec(tuple, &self.class.on_receive_block)
+      if output && self.class.emit?
+        values_list = !output.is_a?(Array) ? [[output]] : !output.first.is_a?(Array) ? [output] : output
+        values_list.each{|values| self.class.anchor? ? anchored_emit(tuple, *values) : unanchored_emit(*values)}
+        @collector.ack(tuple) if self.class.ack?
+      end
+    end
+
+    def prepare(config, context, collector)
+      @collector = collector
+      @context = context
+      @config = config
+      instance_exec(&self.class.on_init_block)
+    end
+
+    def cleanup
+      instance_exec(&self.class.close_block)
+    end
+
+    def declare_output_fields(declarer)
+      declarer.declare(Fields.new(self.class.fields))
+    end
+
+    def get_component_configuration
+      configurator = Configurator.new
+      configurator.instance_exec(&self.class.configure_block)
+      configurator.config
+    end
+
+    private
+
+    # default noop optional dsl callbacks
+    def on_init; end
+    def on_close; end
+
+    def self.fields
+      @fields ||= []
+    end
+
+    def self.configure_block
+      @configure_block ||= lambda {}
+    end
+
+    def self.on_receive_block
+      @on_receive_block ||= lambda {|tuple| self.send(:on_receive, tuple)}
+    end
+
+    def self.on_init_block
+      @on_init_block ||= lambda {self.send(:on_init)}
+    end
+
+    def self.close_block
+      @close_block ||= lambda {self.send(:on_close)}
+    end
+
+    def self.receive_options
+      @receive_options ||= {:emit => true, :ack => false, :anchor => false}
+    end
+
+    def self.emit?
+      !!self.receive_options[:emit]
+    end
+
+    def self.ack?
+      !!self.receive_options[:ack]
+    end
+
+    def self.anchor?
+      !!self.receive_options[:anchor]
+    end
+  end
+end