redstorm 0.6.5 → 0.6.6.beta1

data/lib/red_storm/dsl/spout.rb

@@ -0,0 +1,194 @@
+require 'java'
+require 'red_storm/configurator'
+require 'red_storm/environment'
+require 'pathname'
+
+module RedStorm
+  module DSL
+
+    class SpoutError < StandardError; end
+
+    class Spout
+      attr_reader :config, :context, :collector
+
+      # DSL class methods
+
+      def self.configure(&configure_block)
+        @configure_block = block_given? ? configure_block : lambda {}
+      end
+
+      def self.log
+        @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+      end
+
+      def self.output_fields(*fields)
+        @fields = fields.map(&:to_s)
+      end
+
+      def self.on_send(*args, &on_send_block)
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        method_name = args.first
+
+        self.send_options.merge!(options)
+
+        # indirecting through a lambda defers the method lookup at invocation time
+        # and the performance penalty is negligible
+        body = block_given? ? on_send_block : lambda{self.send((method_name || :on_send).to_sym)}
+        define_method(:on_send, body)
+      end
+
+      def self.on_init(method_name = nil, &on_init_block)
+        body = block_given? ? on_init_block : lambda {self.send((method_name || :on_init).to_sym)}
+        define_method(:on_init, body)
+      end
+
+      def self.on_close(method_name = nil, &on_close_block)
+        body = block_given? ? on_close_block : lambda {self.send((method_name || :on_close).to_sym)}
+        define_method(:on_close, body)
+      end
+
+      def self.on_activate(method_name = nil, &on_activate_block)
+        # @on_activate_block = block_given? ? on_activate_block : lambda {self.send(method_name || :on_activate)}
+        body = block_given? ? on_activate_block : lambda {self.send((method_name || :on_activate).to_sym)}
+        define_method(:on_activate, body)
+      end
+
+      def self.on_deactivate(method_name = nil, &on_deactivate_block)
+        # @on_deactivate_block = block_given? ? on_deactivate_block : lambda {self.send(method_name || :on_deactivate)}
+        body = block_given? ? on_deactivate_block : lambda {self.send((method_name || :on_deactivate).to_sym)}
+        define_method(:on_deactivate, body)
+      end
+
+      def self.on_ack(method_name = nil, &on_ack_block)
+        body = block_given? ? on_ack_block : lambda {|msg_id| self.send((method_name || :on_ack).to_sym, msg_id)}
+        define_method(:on_ack, body)
+      end
+
+      def self.on_fail(method_name = nil, &on_fail_block)
+        body = block_given? ? on_fail_block : lambda {|msg_id| self.send((method_name || :on_fail).to_sym, msg_id)}
+        define_method(:on_fail, body)
+      end
+
+      # DSL instance methods
+
+      def reliable_emit(message_id, *values)
+        @collector.emit(Values.new(*values), message_id)
+      end
+
+      def unreliable_emit(*values)
+        @collector.emit(Values.new(*values))
+      end
+      alias_method :emit, :unreliable_emit
+
+      def log
+        self.class.log
+      end
+
+      # Spout proxy interface
+
+      def next_tuple
+        output = on_send
+
+        if self.class.emit?
+          if output
+            values = [output].flatten
+            if self.class.reliable?
+              message_id = values.shift
+              reliable_emit(message_id, *values)
+            else
+              unreliable_emit(*values)
+            end
+          else
+            sleep(0.1)
+          end
+        end
+      end
+
+      def open(config, context, collector)
+        @collector = collector
+        @context = context
+        @config = config
+
+        on_init
+      end
+
+      def close
+        on_close
+      end
+
+      def activate
+        on_activate
+      end
+
+      def deactivate
+        on_deactivate
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new(self.class.fields))
+      end
+
+      def ack(msg_id)
+        on_ack(msg_id)
+      end
+
+      def fail(msg_id)
+        on_fail(msg_id)
+      end
+
+      def get_component_configuration
+        configurator = Configurator.new
+        configurator.instance_exec(&self.class.configure_block)
+        configurator.config
+      end
+
+      private
+
+      # default optional noop dsl methods/callbacks
+      def on_init; end
+      def on_close; end
+      def on_activate; end
+      def on_deactivate; end
+      def on_ack(msg_id); end
+      def on_fail(msg_id); end
+
+      def self.fields
+        @fields ||= []
+      end
+
+      def self.configure_block
+        @configure_block ||= lambda {}
+      end
+
+      def self.send_options
+        @send_options ||= {:emit => true, :reliable => false}
+      end
+
+      def self.emit?
+        !!self.send_options[:emit]
+      end
+
+      def self.reliable?
+        !!self.send_options[:reliable]
+      end
+
+      # below non-dry see Bolt class
+      def self.inherited(subclass)
+        path = (caller.first.to_s =~ /^(.+):\d+.*$/) ? $1 : raise(SpoutError, "unable to extract base topology class path from #{caller.first.inspect}")
+        subclass.base_class_path = Pathname.new(path).relative_path_from(Pathname.new(RedStorm::BASE_PATH)).to_s
+      end
+
+      def self.base_class_path=(path)
+        @base_class_path = path
+      end
+
+      def self.base_class_path
+        @base_class_path
+      end
+    end
+  end
+
+  # for backward compatibility
+  SimpleSpout = DSL::Spout
+
+end

data/lib/red_storm/dsl/topology.rb

@@ -0,0 +1,227 @@
+require 'java'
+require 'red_storm/configuration'
+require 'red_storm/configurator'
+
+java_import 'backtype.storm.topology.TopologyBuilder'
+
+module RedStorm
+  module DSL
+
+    class TopologyDefinitionError < StandardError; end
+
+    class Topology
+      attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
+
+      DEFAULT_SPOUT_PARALLELISM = 1
+      DEFAULT_BOLT_PARALLELISM = 1
+
+      class ComponentDefinition < Configurator
+        attr_reader :clazz, :constructor_args, :parallelism
+        attr_accessor :id # ids are forced to string
+
+        def initialize(component_class, constructor_args, id, parallelism)
+          super()
+          @clazz = component_class
+          @constructor_args = constructor_args
+          @id = id.to_s
+          @parallelism = parallelism
+          @output_fields = []
+        end
+
+        def output_fields(*args)
+          args.empty? ? @output_fields : @output_fields = args.map(&:to_s)
+        end
+
+        def is_java?
+          @clazz.name.split('::').first.downcase == 'java'
+        end
+      end
+
+      class SpoutDefinition < ComponentDefinition
+
+        # WARNING non-dry see BoltDefinition#new_instance
+        def new_instance
+          if @clazz.name == "Java::RedstormStormJruby::JRubyShellSpout"
+            @clazz.new(constructor_args, @output_fields)
+          elsif is_java?
+            @clazz.new(*constructor_args)
+          else
+            JRubySpout.new(@clazz.base_class_path, @clazz.name, @output_fields)
+          end
+        end
+      end
+
+      class BoltDefinition < ComponentDefinition
+        attr_accessor :sources, :command
+
+        def initialize(*args)
+          super
+          @sources = []
+        end
+
+        def source(source_id, grouping)
+          @sources << [source_id.is_a?(Class) ? Topology.underscore(source_id) : source_id.to_s, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
+        end
+
+        def define_grouping(declarer)
+          @sources.each do |source_id, grouping|
+            grouper, params = grouping.first
+              # declarer.fieldsGrouping(source_id, Fields.new())
+            case grouper
+            when :fields
+              declarer.fieldsGrouping(source_id, Fields.new(*([params].flatten.map(&:to_s))))
+            when :global
+              declarer.globalGrouping(source_id)
+            when :shuffle
+              declarer.shuffleGrouping(source_id)
+            when :local_or_shuffle
+              declarer.localOrShuffleGrouping(source_id)
+            when :none
+              declarer.noneGrouping(source_id)
+            when :all
+              declarer.allGrouping(source_id)
+            when :direct
+              declarer.directGrouping(source_id)
+            else
+              raise("unknown grouper=#{grouper.inspect}")
+            end
+          end
+        end
+
+        def new_instance
+          # WARNING non-dry see BoltDefinition#new_instance
+          if @clazz.name == "Java::RedstormStormJruby::JRubyShellBolt"
+            @clazz.new(constructor_args, @output_fields)
+          elsif is_java?
+            @clazz.new(*constructor_args)
+          else
+            JRubyBolt.new(@clazz.base_class_path, @clazz.name, @output_fields)
+          end
+        end
+      end
+
+      def self.log
+        @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+      end
+
+      # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
+      def self.spout(spout_class, *args, &spout_block)
+        set_topology_class!
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        contructor_args = !args.empty? ? args.pop : []
+        spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
+
+        spout = SpoutDefinition.new(spout_class, contructor_args, spout_options[:id], spout_options[:parallelism])
+        spout.instance_exec(&spout_block) if block_given?
+        self.components << spout
+      end
+
+      # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
+      def self.bolt(bolt_class, *args, &bolt_block)
+        set_topology_class!
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        contructor_args = !args.empty? ? args.pop : []
+        bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+
+        bolt = BoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
+        raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
+        bolt.instance_exec(&bolt_block)
+        self.components << bolt
+      end
+
+      def self.configure(name = nil, &configure_block)
+        set_topology_class!
+        @topology_name = name.to_s if name
+        @configure_block = configure_block if block_given?
+      end
+
+      def self.on_submit(method_name = nil, &submit_block)
+        @submit_block = block_given? ? submit_block : lambda {|env| self.send(method_name, env)}
+      end
+
+      def self.build_topology
+        resolve_ids!(components)
+
+        builder = TopologyBuilder.new
+        spouts.each do |spout|
+          declarer = builder.setSpout(spout.id, spout.new_instance, spout.parallelism.to_java)
+          declarer.addConfigurations(spout.config)
+        end
+        bolts.each do |bolt|
+          declarer = builder.setBolt(bolt.id, bolt.new_instance, bolt.parallelism.to_java)
+          declarer.addConfigurations(bolt.config)
+          bolt.define_grouping(declarer)
+        end
+        builder.createTopology
+      end
+
+      def start(env)
+        topology = self.class.build_topology
+
+        # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
+        defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
+
+        configurator = Configurator.new(defaults)
+        configurator.instance_exec(env, &self.class.configure_block)
+
+        submitter = (env == :local) ? @cluster = LocalCluster.new : StormSubmitter
+        submitter.submitTopology(self.class.topology_name, configurator.config, topology)
+        instance_exec(env, &self.class.submit_block)
+      end
+
+      private
+
+      # this is a quirk to figure out the topology class at load time when the topology file
+      # is required in the TopologyLauncher. Since we want to make the "configure" DSL statement
+      # optional we can hook into any/all the other DSL statements that will be called at load time
+      # and set it there. This is somewhat inelegant but it works.
+      def self.set_topology_class!
+        Configuration.topology_class = self
+      end
+
+      def self.resolve_ids!(components)
+        # verify duplicate implicit ids
+        ids = components.map(&:id)
+        components.reverse.each do |component|
+          raise("duplicate id in #{component.clazz.name} on id=#{component.id}") if ids.select{|id| id == component.id}.size > 1
+          # verify source_id references
+          if component.respond_to?(:sources)
+            component.sources.each{|source_id, grouping| raise("cannot resolve #{component.clazz.name} source id=#{source_id}") unless ids.include?(source_id)}
+          end
+        end
+      end
+
+      def self.spouts
+        self.components.select{|c| c.is_a?(SpoutDefinition)}
+      end
+
+      def self.bolts
+        self.components.select{|c| c.is_a?(BoltDefinition)}
+      end
+
+      def self.components
+        @components ||= []
+      end
+
+      def self.topology_name
+        @topology_name ||= self.underscore(self.name)
+      end
+
+      def self.configure_block
+        @configure_block ||= lambda{|env|}
+      end
+
+      def self.submit_block
+        @submit_block ||= lambda{|env|}
+      end
+
+      def self.underscore(camel_case)
+        camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
+      end
+    end
+  end
+
+  # for backward compatibility
+  SimpleTopology = DSL::Topology
+
+end

data/lib/red_storm/dsl/tuple.rb

@@ -0,0 +1,34 @@
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'backtype.storm.tuple.TupleImpl'
+
+module RedStorm
+  module DSL
+    class TupleError < StandardError; end
+  end
+end
+
+class TupleImpl
+
+  def value(i)
+    case i
+    when Fixnum
+      getValue(i)
+    when String
+      getValueByField(i)
+    when Symbol
+      getValueByField(i.to_s)
+    else
+      raise(RedStorm::DSL::TupleError, "unsupported tuple index class=#{i.class.to_s} for #{i.inspect}")
+    end
+  end
+  alias_method :[], :value
+
+  def field_index(field)
+    fieldIndex(field.to_s)
+  end
+
+  def contains?(field)
+    contains(field.to_s)
+  end
+
+end