kb-redstorm 0.6.4

data/lib/red_storm/simple_drpc_topology.rb

@@ -0,0 +1,87 @@
+require 'java'
+require 'red_storm/configuration'
+require 'red_storm/configurator'
+
+module RedStorm
+
+  class InputBoltDefinition < SimpleTopology::BoltDefinition
+    attr_accessor :grouping
+
+    def initialize(*args)
+      super
+      @grouping = :none
+    end
+
+    def grouping(grouping)
+      @grouping = @grouping
+    end
+
+    def define_grouping(declarer)
+
+      case @grouping
+      when :fields
+        declarer.fieldsGrouping(Fields.new(*([params].flatten.map(&:to_s))))
+      when :global
+        declarer.globalGrouping()
+      when :shuffle
+        declarer.shuffleGrouping()
+      when :local_or_shuffle
+        declarer.localOrShuffleGrouping()
+      when :none
+        declarer.noneGrouping()
+      when :all
+        declarer.allGrouping()
+      when :direct
+        declarer.directGrouping()
+      else
+        raise("unknown grouper=#{grouper.inspect}")
+      end
+    end
+  end
+
+  class SimpleDRPCTopology < SimpleTopology
+
+    def self.spout
+      raise TopologyDefinitionError, "DRPC spout is already defined"
+    end
+
+    def start(base_class_path, env)
+      builder = Java::BacktypeStormDrpc::LinearDRPCTopologyBuilder.new(self.class.topology_name)
+
+      self.class.bolts.each do |bolt|
+        declarer = builder.addBolt(bolt.new_instance(base_class_path), bolt.parallelism.to_java)
+        declarer.addConfigurations(bolt.config)
+        bolt.define_grouping(declarer)
+      end
+
+      # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
+      defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
+
+      configurator = Configurator.new(defaults)
+      configurator.instance_exec(env, &self.class.configure_block)
+
+      drpc = nil
+      if env == :local
+        drpc = LocalDRPC.new
+        submitter = @cluster = LocalCluster.new
+        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createLocalTopology(drpc))
+      else
+        submitter = StormSubmitter
+        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createRemoteTopology)
+      end
+      instance_exec(env, drpc, &self.class.submit_block)
+    end
+
+    def self.input_bolt(bolt_class, *args, &bolt_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
+      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+
+      bolt = InputBoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
+      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
+      bolt.instance_exec(&bolt_block)
+      self.components << bolt
+    end
+  end
+
+end

data/lib/red_storm/simple_spout.rb

@@ -0,0 +1,184 @@
+require 'java'
+require 'red_storm/configurator'
+
+module RedStorm
+
+  class SimpleSpout
+    attr_reader :config, :context, :collector
+
+    # DSL class methods
+
+    def self.configure(&configure_block)
+      @configure_block = block_given? ? configure_block : lambda {}
+    end
+
+    def self.log
+      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+    end
+
+    def self.output_fields(*fields)
+      @fields = fields.map(&:to_s)
+    end
+
+    def self.on_send(*args, &on_send_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      method_name = args.first
+      
+      self.send_options.merge!(options)
+      @on_send_block = block_given? ? on_send_block : lambda {self.send(method_name || :on_send)}
+    end
+
+    def self.on_init(method_name = nil, &on_init_block)
+      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
+    end
+
+    def self.on_close(method_name = nil, &on_close_block)
+      @on_close_block = block_given? ? on_close_block : lambda {self.send(method_name || :on_close)}
+    end
+
+    def self.on_activate(method_name = nil, &on_activate_block)
+      @on_activate_block = block_given? ? on_activate_block : lambda {self.send(method_name || :on_activate)}
+    end
+
+    def self.on_deactivate(method_name = nil, &on_deactivate_block)
+      @on_deactivate_block = block_given? ? on_deactivate_block : lambda {self.send(method_name || :on_deactivate)}
+    end
+
+    def self.on_ack(method_name = nil, &on_ack_block)
+      @on_ack_block = block_given? ? on_ack_block : lambda {|msg_id| self.send(method_name || :on_ack, msg_id)}
+    end
+
+    def self.on_fail(method_name = nil, &on_fail_block)
+      @on_fail_block = block_given? ? on_fail_block : lambda {|msg_id| self.send(method_name || :on_fail, msg_id)}
+    end
+
+    # DSL instance methods
+
+    def reliable_emit(message_id, *values)
+      @collector.emit(Values.new(*values), message_id) 
+    end
+
+    def unreliable_emit(*values)
+      @collector.emit(Values.new(*values)) 
+    end
+    alias_method :emit, :unreliable_emit
+
+    def log
+      self.class.log
+    end
+
+    # Spout proxy interface
+
+    def next_tuple
+      output = instance_exec(&self.class.on_send_block)
+      if self.class.emit?
+        if output
+          values = [output].flatten
+          if self.class.reliable?
+            message_id = values.shift
+            reliable_emit(message_id, *values)
+          else
+            unreliable_emit(*values)
+          end
+        else
+          sleep(0.1)
+        end
+      end
+    end
+
+    def open(config, context, collector)
+      @collector = collector
+      @context = context
+      @config = config
+      instance_exec(&self.class.on_init_block)
+    end
+
+    def close
+      instance_exec(&self.class.on_close_block)
+    end
+
+    def activate
+      instance_exec(&self.class.on_activate_block)
+    end
+
+    def deactivate
+      instance_exec(&self.class.on_deactivate_block)
+    end
+
+    def declare_output_fields(declarer)
+      declarer.declare(Fields.new(self.class.fields))
+    end
+
+    def ack(msg_id)
+      instance_exec(msg_id, &self.class.on_ack_block)
+    end
+
+    def fail(msg_id)
+      instance_exec(msg_id, &self.class.on_fail_block)
+    end
+
+    def get_component_configuration
+      configurator = Configurator.new
+      configurator.instance_exec(&self.class.configure_block)
+      configurator.config
+    end
+
+    private
+
+    # default optional noop dsl methods/callbacks
+    def on_init; end
+    def on_close; end
+    def on_activate; end
+    def on_deactivate; end
+    def on_ack(msg_id); end
+    def on_fail(msg_id); end
+
+    def self.fields
+      @fields ||= []
+    end
+
+    def self.configure_block
+      @configure_block ||= lambda {}
+    end
+
+    def self.on_send_block
+      @on_send_block ||= lambda {self.send(:on_send)}
+    end
+
+    def self.on_init_block
+      @on_init_block ||= lambda {self.send(:on_init)}
+    end
+
+    def self.on_close_block
+      @on_close_block ||= lambda {self.send(:on_close)}
+    end
+
+    def self.on_activate_block
+      @on_activate_block ||= lambda {self.send(:on_activate)}
+    end
+
+    def self.on_deactivate_block
+      @on_deactivate_block ||= lambda {self.send(:on_deactivate)}
+    end
+
+    def self.on_ack_block
+      @on_ack_block ||= lambda {|msg_id| self.send(:on_ack, msg_id)}
+    end
+
+    def self.on_fail_block
+      @on_fail_block ||= lambda {|msg_id| self.send(:on_fail, msg_id)}
+    end
+
+    def self.send_options
+      @send_options ||= {:emit => true, :reliable => false}
+    end
+
+    def self.emit?
+      !!self.send_options[:emit]
+    end
+
+    def self.reliable?
+      !!self.send_options[:reliable]
+    end
+  end
+end

data/lib/red_storm/simple_topology.rb

@@ -0,0 +1,209 @@
+require 'java'
+require 'red_storm/configuration'
+require 'red_storm/configurator'
+
+
+module RedStorm
+
+  class TopologyDefinitionError < StandardError; end
+
+  class SimpleTopology
+    attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
+
+    DEFAULT_SPOUT_PARALLELISM = 1
+    DEFAULT_BOLT_PARALLELISM = 1
+
+    class ComponentDefinition < Configurator
+      attr_reader :clazz, :constructor_args, :parallelism
+      attr_accessor :id # ids are forced to string
+
+      def initialize(component_class, constructor_args, id, parallelism)
+        super()
+        @clazz = component_class
+        @constructor_args = constructor_args
+        @id = id.to_s
+        @parallelism = parallelism
+        @output_fields = []
+      end
+
+      def output_fields(*args)
+        args.empty? ? @output_fields : @output_fields = args.map(&:to_s)
+      end
+
+      def is_java?
+        @clazz.name.split('::').first.downcase == 'java'
+      end
+    end
+
+    class SpoutDefinition < ComponentDefinition
+
+      # WARNING non-dry see BoltDefinition#new_instance
+      def new_instance(base_class_path)
+        if @clazz.name == "Java::RedstormStormJruby::JRubyShellSpout"
+          @clazz.new(constructor_args, @output_fields)
+        elsif is_java?
+          @clazz.new(*constructor_args)
+        else
+          JRubySpout.new(base_class_path, @clazz.name, @output_fields)
+        end
+        # is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
+      end
+    end
+
+    class BoltDefinition < ComponentDefinition
+      attr_accessor :sources, :command
+
+      def initialize(*args)
+        super
+        @sources = []
+      end
+
+      def source(source_id, grouping)
+        @sources << [source_id.is_a?(Class) ? SimpleTopology.underscore(source_id) : source_id.to_s, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
+      end
+
+      def define_grouping(declarer)
+        @sources.each do |source_id, grouping|
+          grouper, params = grouping.first
+            # declarer.fieldsGrouping(source_id, Fields.new())
+          case grouper
+          when :fields
+            declarer.fieldsGrouping(source_id, Fields.new(*([params].flatten.map(&:to_s))))
+          when :global
+            declarer.globalGrouping(source_id)
+          when :shuffle
+            declarer.shuffleGrouping(source_id)
+          when :local_or_shuffle
+            declarer.localOrShuffleGrouping(source_id)
+          when :none
+            declarer.noneGrouping(source_id)
+          when :all
+            declarer.allGrouping(source_id)
+          when :direct
+            declarer.directGrouping(source_id)
+          else
+            raise("unknown grouper=#{grouper.inspect}")
+          end
+        end
+      end
+
+      def new_instance(base_class_path)
+        # WARNING non-dry see BoltDefinition#new_instance
+        if @clazz.name == "Java::RedstormStormJruby::JRubyShellBolt"
+          @clazz.new(constructor_args, @output_fields)
+        elsif is_java?
+          @clazz.new(*constructor_args)
+        else
+          JRubyBolt.new(base_class_path, @clazz.name, @output_fields)
+        end
+        # is_java? ? @clazz.new : @clazz.is_a?(SimpleBolt) ? JRubyBolt.new(base_class_path, @clazz.name) : @clazz.new
+      end
+    end
+
+    def self.log
+      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+    end
+
+    # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
+    def self.spout(spout_class, *args, &spout_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
+      spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
+
+      spout = SpoutDefinition.new(spout_class, contructor_args, spout_options[:id], spout_options[:parallelism])
+      spout.instance_exec(&spout_block) if block_given?
+      self.components << spout
+    end
+
+    # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
+    def self.bolt(bolt_class, *args, &bolt_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
+      bolt_options = {:id => options[:id] ? options[:id] : self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+
+      bolt = BoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
+      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
+      bolt.instance_exec(&bolt_block)
+      self.components << bolt
+    end
+
+    def self.configure(name = nil, &configure_block)
+      Configuration.topology_class = self
+      @topology_name = name if name
+      @configure_block = configure_block if block_given?
+    end
+
+    def self.on_submit(method_name = nil, &submit_block)
+      @submit_block = block_given? ? submit_block : lambda {|env| self.send(method_name, env)}
+    end
+
+    # topology proxy interface
+
+    def start(base_class_path, env)
+      self.class.resolve_ids!(self.class.components)
+
+      builder = TopologyBuilder.new
+      self.class.spouts.each do |spout|
+        declarer = builder.setSpout(spout.id, spout.new_instance(base_class_path), spout.parallelism.to_java)
+        declarer.addConfigurations(spout.config)
+      end
+      self.class.bolts.each do |bolt|
+        declarer = builder.setBolt(bolt.id, bolt.new_instance(base_class_path), bolt.parallelism.to_java)
+        declarer.addConfigurations(bolt.config)
+        bolt.define_grouping(declarer)
+      end
+
+      # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
+      defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
+
+      configurator = Configurator.new(defaults)
+      configurator.instance_exec(env, &self.class.configure_block)
+
+      submitter = (env == :local) ? @cluster = LocalCluster.new : StormSubmitter
+      submitter.submitTopology(self.class.topology_name, configurator.config, builder.createTopology)
+      instance_exec(env, &self.class.submit_block)
+    end
+
+    private
+
+    def self.resolve_ids!(components)
+      # verify duplicate implicit ids
+      ids = components.map(&:id)
+      components.reverse.each do |component|
+        raise("duplicate id in #{component.clazz.name} on id=#{component.id}") if ids.select{|id| id == component.id}.size > 1
+        # verify source_id references
+        if component.respond_to?(:sources)
+          component.sources.each{|source_id, grouping| raise("cannot resolve #{component.clazz.name} source id=#{source_id}") unless ids.include?(source_id)}
+        end
+      end
+    end
+
+    def self.spouts
+      self.components.select{|c| c.is_a?(SpoutDefinition)}
+    end
+
+    def self.bolts
+      self.components.select{|c| c.is_a?(BoltDefinition)}
+    end
+
+    def self.components
+      @components ||= []
+    end
+
+    def self.topology_name
+      @topology_name ||= self.underscore(self.name)
+    end
+
+    def self.configure_block
+      @configure_block ||= lambda{|env|}
+    end
+
+    def self.submit_block
+      @submit_block ||= lambda{|env|}
+    end
+
+    def self.underscore(camel_case)
+      camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
+    end
+  end
+end