redstorm 0.6.5 → 0.6.6.beta1

data/lib/red_storm/simple_drpc_topology.rb

@@ -1,87 +0,0 @@
-require 'java'
-require 'red_storm/configuration'
-require 'red_storm/configurator'
-
-module RedStorm
-
-  class InputBoltDefinition < SimpleTopology::BoltDefinition
-    attr_accessor :grouping
-
-    def initialize(*args)
-      super
-      @grouping = :none
-    end
-
-    def grouping(grouping)
-      @grouping = grouping
-    end
-
-    def define_grouping(declarer)
-      case @grouping
-      when :fields
-        declarer.fieldsGrouping(Fields.new(*([params].flatten.map(&:to_s))))
-      when :global
-        declarer.globalGrouping()
-      when :shuffle
-        declarer.shuffleGrouping()
-      when :local_or_shuffle
-        declarer.localOrShuffleGrouping()
-      when :none
-        declarer.noneGrouping()
-      when :all
-        declarer.allGrouping()
-      when :direct
-        declarer.directGrouping()
-      else
-        raise("unknown grouper=#{grouper.inspect}")
-      end
-    end
-  end
-
-  class SimpleDRPCTopology < SimpleTopology
-
-    def self.spout
-      raise TopologyDefinitionError, "DRPC spout is already defined"
-    end
-
-    def start(base_class_path, env)
-      builder = Java::BacktypeStormDrpc::LinearDRPCTopologyBuilder.new(self.class.topology_name)
-
-      self.class.bolts.each do |bolt|
-        declarer = builder.addBolt(bolt.new_instance(base_class_path), bolt.parallelism.to_java)
-        declarer.addConfigurations(bolt.config)
-        bolt.define_grouping(declarer)
-      end
-
-      # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
-      defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
-
-      configurator = Configurator.new(defaults)
-      configurator.instance_exec(env, &self.class.configure_block)
-
-      drpc = nil
-      if env == :local
-        drpc = LocalDRPC.new
-        submitter = @cluster = LocalCluster.new
-        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createLocalTopology(drpc))
-      else
-        submitter = StormSubmitter
-        submitter.submitTopology(self.class.topology_name, configurator.config, builder.createRemoteTopology)
-      end
-      instance_exec(env, drpc, &self.class.submit_block)
-    end
-
-    def self.input_bolt(bolt_class, *args, &bolt_block)
-      set_topology_class!
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      contructor_args = !args.empty? ? args.pop : []
-      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
-
-      bolt = InputBoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
-      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
-      bolt.instance_exec(&bolt_block)
-      self.components << bolt
-    end
-  end
-
-end

data/lib/red_storm/simple_spout.rb

@@ -1,184 +0,0 @@
-require 'java'
-require 'red_storm/configurator'
-
-module RedStorm
-
-  class SimpleSpout
-    attr_reader :config, :context, :collector
-
-    # DSL class methods
-
-    def self.configure(&configure_block)
-      @configure_block = block_given? ? configure_block : lambda {}
-    end
-
-    def self.log
-      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
-    end
-
-    def self.output_fields(*fields)
-      @fields = fields.map(&:to_s)
-    end
-
-    def self.on_send(*args, &on_send_block)
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      method_name = args.first
-      
-      self.send_options.merge!(options)
-      @on_send_block = block_given? ? on_send_block : lambda {self.send(method_name || :on_send)}
-    end
-
-    def self.on_init(method_name = nil, &on_init_block)
-      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
-    end
-
-    def self.on_close(method_name = nil, &on_close_block)
-      @on_close_block = block_given? ? on_close_block : lambda {self.send(method_name || :on_close)}
-    end
-
-    def self.on_activate(method_name = nil, &on_activate_block)
-      @on_activate_block = block_given? ? on_activate_block : lambda {self.send(method_name || :on_activate)}
-    end
-
-    def self.on_deactivate(method_name = nil, &on_deactivate_block)
-      @on_deactivate_block = block_given? ? on_deactivate_block : lambda {self.send(method_name || :on_deactivate)}
-    end
-
-    def self.on_ack(method_name = nil, &on_ack_block)
-      @on_ack_block = block_given? ? on_ack_block : lambda {|msg_id| self.send(method_name || :on_ack, msg_id)}
-    end
-
-    def self.on_fail(method_name = nil, &on_fail_block)
-      @on_fail_block = block_given? ? on_fail_block : lambda {|msg_id| self.send(method_name || :on_fail, msg_id)}
-    end
-
-    # DSL instance methods
-
-    def reliable_emit(message_id, *values)
-      @collector.emit(Values.new(*values), message_id) 
-    end
-
-    def unreliable_emit(*values)
-      @collector.emit(Values.new(*values)) 
-    end
-    alias_method :emit, :unreliable_emit
-
-    def log
-      self.class.log
-    end
-
-    # Spout proxy interface
-
-    def next_tuple
-      output = instance_exec(&self.class.on_send_block)
-      if self.class.emit?
-        if output
-          values = [output].flatten
-          if self.class.reliable?
-            message_id = values.shift
-            reliable_emit(message_id, *values)
-          else
-            unreliable_emit(*values)
-          end
-        else
-          sleep(0.1)
-        end
-      end
-    end
-
-    def open(config, context, collector)
-      @collector = collector
-      @context = context
-      @config = config
-      instance_exec(&self.class.on_init_block)
-    end
-
-    def close
-      instance_exec(&self.class.on_close_block)
-    end
-
-    def activate
-      instance_exec(&self.class.on_activate_block)
-    end
-
-    def deactivate
-      instance_exec(&self.class.on_deactivate_block)
-    end
-
-    def declare_output_fields(declarer)
-      declarer.declare(Fields.new(self.class.fields))
-    end
-
-    def ack(msg_id)
-      instance_exec(msg_id, &self.class.on_ack_block)
-    end
-
-    def fail(msg_id)
-      instance_exec(msg_id, &self.class.on_fail_block)
-    end
-
-    def get_component_configuration
-      configurator = Configurator.new
-      configurator.instance_exec(&self.class.configure_block)
-      configurator.config
-    end
-
-    private
-
-    # default optional noop dsl methods/callbacks
-    def on_init; end
-    def on_close; end
-    def on_activate; end
-    def on_deactivate; end
-    def on_ack(msg_id); end
-    def on_fail(msg_id); end
-
-    def self.fields
-      @fields ||= []
-    end
-
-    def self.configure_block
-      @configure_block ||= lambda {}
-    end
-
-    def self.on_send_block
-      @on_send_block ||= lambda {self.send(:on_send)}
-    end
-
-    def self.on_init_block
-      @on_init_block ||= lambda {self.send(:on_init)}
-    end
-
-    def self.on_close_block
-      @on_close_block ||= lambda {self.send(:on_close)}
-    end
-
-    def self.on_activate_block
-      @on_activate_block ||= lambda {self.send(:on_activate)}
-    end
-
-    def self.on_deactivate_block
-      @on_deactivate_block ||= lambda {self.send(:on_deactivate)}
-    end
-
-    def self.on_ack_block
-      @on_ack_block ||= lambda {|msg_id| self.send(:on_ack, msg_id)}
-    end
-
-    def self.on_fail_block
-      @on_fail_block ||= lambda {|msg_id| self.send(:on_fail, msg_id)}
-    end
-
-    def self.send_options
-      @send_options ||= {:emit => true, :reliable => false}
-    end
-
-    def self.emit?
-      !!self.send_options[:emit]
-    end
-
-    def self.reliable?
-      !!self.send_options[:reliable]
-    end
-  end
-end

data/lib/red_storm/simple_topology.rb

@@ -1,219 +0,0 @@
-require 'java'
-require 'red_storm/configuration'
-require 'red_storm/configurator'
-
-
-module RedStorm
-
-  class TopologyDefinitionError < StandardError; end
-
-  class SimpleTopology
-    attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
-
-    DEFAULT_SPOUT_PARALLELISM = 1
-    DEFAULT_BOLT_PARALLELISM = 1
-
-    class ComponentDefinition < Configurator
-      attr_reader :clazz, :constructor_args, :parallelism
-      attr_accessor :id # ids are forced to string
-
-      def initialize(component_class, constructor_args, id, parallelism)
-        super()
-        @clazz = component_class
-        @constructor_args = constructor_args
-        @id = id.to_s
-        @parallelism = parallelism
-        @output_fields = []
-      end
-
-      def output_fields(*args)
-        args.empty? ? @output_fields : @output_fields = args.map(&:to_s)
-      end
-
-      def is_java?
-        @clazz.name.split('::').first.downcase == 'java'
-      end
-    end
-
-    class SpoutDefinition < ComponentDefinition
-
-      # WARNING non-dry see BoltDefinition#new_instance
-      def new_instance(base_class_path)
-        if @clazz.name == "Java::RedstormStormJruby::JRubyShellSpout"
-          @clazz.new(constructor_args, @output_fields)
-        elsif is_java?
-          @clazz.new(*constructor_args)
-        else
-          JRubySpout.new(base_class_path, @clazz.name, @output_fields)
-        end
-        # is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
-      end
-    end
-
-    class BoltDefinition < ComponentDefinition
-      attr_accessor :sources, :command
-
-      def initialize(*args)
-        super
-        @sources = []
-      end
-
-      def source(source_id, grouping)
-        @sources << [source_id.is_a?(Class) ? SimpleTopology.underscore(source_id) : source_id.to_s, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
-      end
-
-      def define_grouping(declarer)
-        @sources.each do |source_id, grouping|
-          grouper, params = grouping.first
-            # declarer.fieldsGrouping(source_id, Fields.new())
-          case grouper
-          when :fields
-            declarer.fieldsGrouping(source_id, Fields.new(*([params].flatten.map(&:to_s))))
-          when :global
-            declarer.globalGrouping(source_id)
-          when :shuffle
-            declarer.shuffleGrouping(source_id)
-          when :local_or_shuffle
-            declarer.localOrShuffleGrouping(source_id)
-          when :none
-            declarer.noneGrouping(source_id)
-          when :all
-            declarer.allGrouping(source_id)
-          when :direct
-            declarer.directGrouping(source_id)
-          else
-            raise("unknown grouper=#{grouper.inspect}")
-          end
-        end
-      end
-
-      def new_instance(base_class_path)
-        # WARNING non-dry see BoltDefinition#new_instance
-        if @clazz.name == "Java::RedstormStormJruby::JRubyShellBolt"
-          @clazz.new(constructor_args, @output_fields)
-        elsif is_java?
-          @clazz.new(*constructor_args)
-        else
-          JRubyBolt.new(base_class_path, @clazz.name, @output_fields)
-        end
-        # is_java? ? @clazz.new : @clazz.is_a?(SimpleBolt) ? JRubyBolt.new(base_class_path, @clazz.name) : @clazz.new
-      end
-    end
-
-    def self.log
-      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
-    end
-
-    # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
-    def self.spout(spout_class, *args, &spout_block)
-      set_topology_class!
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      contructor_args = !args.empty? ? args.pop : []
-      spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
-
-      spout = SpoutDefinition.new(spout_class, contructor_args, spout_options[:id], spout_options[:parallelism])
-      spout.instance_exec(&spout_block) if block_given?
-      self.components << spout
-    end
-
-    # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
-    def self.bolt(bolt_class, *args, &bolt_block)
-      set_topology_class!
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      contructor_args = !args.empty? ? args.pop : []
-      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
-
-      bolt = BoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
-      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
-      bolt.instance_exec(&bolt_block)
-      self.components << bolt
-    end
-
-    def self.configure(name = nil, &configure_block)
-      set_topology_class!
-      @topology_name = name.to_s if name
-      @configure_block = configure_block if block_given?
-    end
-
-    def self.on_submit(method_name = nil, &submit_block)
-      @submit_block = block_given? ? submit_block : lambda {|env| self.send(method_name, env)}
-    end
-
-    # topology proxy interface
-
-    def start(base_class_path, env)
-      self.class.resolve_ids!(self.class.components)
-
-      builder = TopologyBuilder.new
-      self.class.spouts.each do |spout|
-        declarer = builder.setSpout(spout.id, spout.new_instance(base_class_path), spout.parallelism.to_java)
-        declarer.addConfigurations(spout.config)
-      end
-      self.class.bolts.each do |bolt|
-        declarer = builder.setBolt(bolt.id, bolt.new_instance(base_class_path), bolt.parallelism.to_java)
-        declarer.addConfigurations(bolt.config)
-        bolt.define_grouping(declarer)
-      end
-
-      # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
-      defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
-
-      configurator = Configurator.new(defaults)
-      configurator.instance_exec(env, &self.class.configure_block)
- 
-      submitter = (env == :local) ? @cluster = LocalCluster.new : StormSubmitter
-      submitter.submitTopology(self.class.topology_name, configurator.config, builder.createTopology)
-      instance_exec(env, &self.class.submit_block)
-    end
-
-    private
-
-    # this is a quirk to figure out the topology class at load time when the topology file
-    # is required in the TopologyLauncher. Since we want to make the "configure" DSL statement 
-    # optional we can hook into any/all the other DSL statements that will be called at load time
-    # and set it there. This is somewhat inelegant but it works. 
-    def self.set_topology_class!
-      Configuration.topology_class = self
-    end
-
-    def self.resolve_ids!(components)
-      # verify duplicate implicit ids
-      ids = components.map(&:id)
-      components.reverse.each do |component|
-        raise("duplicate id in #{component.clazz.name} on id=#{component.id}") if ids.select{|id| id == component.id}.size > 1
-        # verify source_id references
-        if component.respond_to?(:sources)
-          component.sources.each{|source_id, grouping| raise("cannot resolve #{component.clazz.name} source id=#{source_id}") unless ids.include?(source_id)}
-        end
-      end
-    end
-
-    def self.spouts
-      self.components.select{|c| c.is_a?(SpoutDefinition)}
-    end
-
-    def self.bolts
-      self.components.select{|c| c.is_a?(BoltDefinition)}
-    end
-
-    def self.components
-      @components ||= []
-    end
-
-    def self.topology_name
-      @topology_name ||= self.underscore(self.name)
-    end
-
-    def self.configure_block
-      @configure_block ||= lambda{|env|}
-    end
-
-    def self.submit_block
-      @submit_block ||= lambda{|env|}
-    end
-
-    def self.underscore(camel_case)
-      camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
-    end
-  end
-end