redstorm 0.6.4 → 0.6.5

data/lib/red_storm/proxy/batch_bolt.rb

@@ -0,0 +1,63 @@
+require 'java'
+
+java_import 'backtype.storm.coordination.BatchOutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.topology.IRichBolt'
+java_import 'backtype.storm.coordination.IBatchBolt'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the Bolt class is a proxy to the real bolt to avoid having to deal with all the
+# Java artifacts when creating a bolt.
+#
+# The real bolt class implementation must define these methods:
+# - prepare(conf, context, collector)
+# - execute(tuple)
+# - declare_output_fields
+#
+# and optionnaly:
+# - cleanup
+#
+class BatchBolt
+  java_implements IBatchBolt
+
+  java_signature 'IBatchBolt (String base_class_path, String real_bolt_class_name)'
+  def initialize(base_class_path, real_bolt_class_name)
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  end
+
+  java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
+  def prepare(conf, context, collector, id)
+    @real_bolt.prepare(conf, context, collector, id)
+  end
+
+  java_signature 'void execute(Tuple)'
+  def execute(tuple)
+    @real_bolt.execute(tuple)
+  end
+
+  java_signature 'void finishBatch()'
+  def finishBatch
+    @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_bolt.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
+end

data/lib/red_storm/proxy/batch_committer_bolt.rb

@@ -0,0 +1,52 @@
+require 'java'
+
+java_import 'backtype.storm.coordination.BatchOutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.coordination.IBatchBolt'
+java_import 'backtype.storm.transactional.ICommitter'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+class BatchCommitterBolt
+  java_implements 'ICommitter, IBatchBolt'
+
+  java_signature 'IBatchCommitterBolt (String base_class_path, String real_bolt_class_name)'
+  def initialize(base_class_path, real_bolt_class_name)
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  end
+
+  java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
+  def prepare(conf, context, collector, id)
+    @real_bolt.prepare(conf, context, collector, id)
+  end
+
+  java_signature 'void execute(Tuple)'
+  def execute(tuple)
+    @real_bolt.execute(tuple)
+  end
+
+  java_signature 'void finishBatch()'
+  def finishBatch
+    @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_bolt.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
+end

data/lib/red_storm/proxy/batch_spout.rb

@@ -0,0 +1,59 @@
+require 'java'
+
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'storm.trident.operation.TridentCollector'
+java_import 'storm.trident.spout.IBatchSpout'
+java_import 'backtype.storm.tuple.Fields'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the Spout class is a proxy to the real spout to avoid having to deal with all the
+# Java artifacts when creating a spout.
+
+class BatchSpout
+  java_implements IBatchSpout
+
+  java_signature 'IBatchSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'void open(Map, TopologyContext)'
+  def open(conf, context)
+    @real_spout.open(conf, context) if @real_spout.respond_to?(:open)
+  end
+
+  java_signature 'void emitBatch(long, TridentCollector)'
+  def emitBatch(batch_id, collector)
+  	@real_spout.emit_batch(batch_id, collector)
+  end
+
+  java_signature 'void close()'
+  def close
+    @real_spout.close if @real_spout.respond_to?(:close)
+  end
+
+  java_signature 'void ack(long)'
+  def ack(batch_id)
+    @real_spout.ack(batch_id) if @real_spout.respond_to?(:ack)
+  end
+
+  java_signature 'Fields getOutputFields()'
+  def getOutputFields()
+    @real_spout.get_output_fields
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/proxy/proxy_function.rb

@@ -0,0 +1,40 @@
+require 'java'
+
+java_import 'storm.trident.tuple.TridentTuple'
+java_import 'storm.trident.operation.TridentCollector'
+java_import 'storm.trident.operation.TridentOperationContext'
+java_import 'storm.trident.operation.Function'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+class ProxyFunction
+  java_implements Function
+
+  java_signature 'Function (String base_class_path, String real_class_name)'
+  def initialize(base_class_path, real_class_name)
+    @real = Object.module_eval(real_class_name).new
+  rescue NameError
+    require base_class_path
+    @real = Object.module_eval(real_class_name).new
+  end
+
+  java_signature 'void execute(TridentTuple, TridentCollector)'
+  def execute(_trident_tuple, _trident_collector)
+    @real.execute(_trident_tuple, _trident_collector)
+  end
+
+  java_signature 'void cleanup()'
+  def cleanup()
+    @real.cleanup()
+  end
+
+  java_signature 'void prepare(Map, TridentOperationContext)'
+  def prepare(_map, _trident_operation_context)
+    @real.prepare(_map, _trident_operation_context)
+  end
+end

data/lib/red_storm/proxy/transactional_committer_spout.rb

@@ -0,0 +1,47 @@
+require 'java'
+
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.transactional.ITransactionalSpout'
+java_import 'backtype.storm.transactional.ICommitterTransactionalSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+
+class TransactionalCommitterSpout
+  java_implements 'ICommitterTransactionalSpout'
+
+  java_signature 'ICommitterTransactionalSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'ICommitterTransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
+  def getEmitter(conf, context)
+    @real_spout.get_emitter(conf, context)
+  end
+
+  java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
+  def getCoordinator(conf, context)
+    @real_spout.get_coordinator(conf, context)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_spout.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/proxy/transactional_spout.rb

@@ -0,0 +1,46 @@
+require 'java'
+
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.transactional.ITransactionalSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+
+class TransactionalSpout
+  java_implements 'ITransactionalSpout'
+
+  java_signature 'ITransactionalSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'ITransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
+  def getEmitter(conf, context)
+    @real_spout.get_emitter(conf, context)
+  end
+
+  java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
+  def getCoordinator(conf, context)
+    @real_spout.get_coordinator(conf, context)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_spout.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/simple_drpc_topology.rb

@@ -0,0 +1,87 @@
+require 'java'
+require 'red_storm/configuration'
+require 'red_storm/configurator'
+
+module RedStorm
+
+  class InputBoltDefinition < SimpleTopology::BoltDefinition
+    attr_accessor :grouping
+
+    def initialize(*args)
+      super
+      @grouping = :none
+    end
+
+    def grouping(grouping)
+      @grouping = grouping
+    end
+
+    def define_grouping(declarer)
+      case @grouping
+      when :fields
+        declarer.fieldsGrouping(Fields.new(*([params].flatten.map(&:to_s))))
+      when :global
+        declarer.globalGrouping()
+      when :shuffle
+        declarer.shuffleGrouping()
+      when :local_or_shuffle
+        declarer.localOrShuffleGrouping()
+      when :none
+        declarer.noneGrouping()
+      when :all
+        declarer.allGrouping()
+      when :direct
+        declarer.directGrouping()
+      else
+        raise("unknown grouper=#{grouper.inspect}")
+      end
+    end
+  end
+
+  class SimpleDRPCTopology < SimpleTopology
+
+    def self.spout
+      raise TopologyDefinitionError, "DRPC spout is already defined"
+    end
+
+    def start(base_class_path, env)
+      builder = Java::BacktypeStormDrpc::LinearDRPCTopologyBuilder.new(self.class.topology_name)
+
+      self.class.bolts.each do |bolt|
+        declarer = builder.addBolt(bolt.new_instance(base_class_path), bolt.parallelism.to_java)
+        declarer.addConfigurations(bolt.config)
+        bolt.define_grouping(declarer)
+      end
+
+      # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
+      defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
+
+      configurator = Configurator.new(defaults)
+      configurator.instance_exec(env, &self.class.configure_block)
+
+      drpc = nil
+      if env == :local
+        drpc = LocalDRPC.new
+        submitter = @cluster = LocalCluster.new
+        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createLocalTopology(drpc))
+      else
+        submitter = StormSubmitter
+        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createRemoteTopology)
+      end
+      instance_exec(env, drpc, &self.class.submit_block)
+    end
+
+    def self.input_bolt(bolt_class, *args, &bolt_block)
+      set_topology_class!
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
+      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+
+      bolt = InputBoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
+      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
+      bolt.instance_exec(&bolt_block)
+      self.components << bolt
+    end
+  end
+
+end

data/lib/red_storm/simple_topology.rb

@@ -36,7 +36,7 @@ module RedStorm
     end
 
     class SpoutDefinition < ComponentDefinition
-      
+
       # WARNING non-dry see BoltDefinition#new_instance
       def new_instance(base_class_path)
         if @clazz.name == "Java::RedstormStormJruby::JRubyShellSpout"
@@ -49,7 +49,7 @@ module RedStorm
         # is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
       end
     end
-          
+
     class BoltDefinition < ComponentDefinition
       attr_accessor :sources, :command
 
@@ -106,6 +106,7 @@ module RedStorm
 
     # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
     def self.spout(spout_class, *args, &spout_block)
+      set_topology_class!
       options = args.last.is_a?(Hash) ? args.pop : {}
       contructor_args = !args.empty? ? args.pop : []
       spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
@@ -117,6 +118,7 @@ module RedStorm
 
     # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
     def self.bolt(bolt_class, *args, &bolt_block)
+      set_topology_class!
       options = args.last.is_a?(Hash) ? args.pop : {}
       contructor_args = !args.empty? ? args.pop : []
       bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
@@ -128,8 +130,8 @@ module RedStorm
     end
 
     def self.configure(name = nil, &configure_block)
-      Configuration.topology_class = self
-      @topology_name = name if name
+      set_topology_class!
+      @topology_name = name.to_s if name
       @configure_block = configure_block if block_given?
     end
 
@@ -166,6 +168,14 @@ module RedStorm
 
     private
 
+    # this is a quirk to figure out the topology class at load time when the topology file
+    # is required in the TopologyLauncher. Since we want to make the "configure" DSL statement 
+    # optional we can hook into any/all the other DSL statements that will be called at load time
+    # and set it there. This is somewhat inelegant but it works. 
+    def self.set_topology_class!
+      Configuration.topology_class = self
+    end
+
     def self.resolve_ids!(components)
       # verify duplicate implicit ids
       ids = components.map(&:id)