redstorm 0.1.1 → 0.2.0

data/lib/red_storm.rb

@@ -4,3 +4,6 @@ end
 
 require 'red_storm/version'
 require 'red_storm/application'
+require 'red_storm/simple_bolt'
+require 'red_storm/simple_spout'
+require 'red_storm/simple_topology'

data/lib/red_storm/application.rb

@@ -1,19 +1,26 @@
-require 'rake'
+module RedStorm
+  
+  class Application 
+    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
 
-class RedStorm::Application 
+    def usage
+      puts("usage: redstorm install|examples|jar <project_directory>|local <topology_class_file>")
+      exit(1)
+    end
 
-  def run(args)
-    if args.size == 1 && File.exist?(args.first)
-      load("#{RedStorm::REDSTORM_HOME}/Rakefile")
-      Rake::Task['launch'].invoke(args)
-    else
-      task = args.shift
-      if ["install", "examples", "jar"].include?(task)
-        load("#{RedStorm::REDSTORM_HOME}/Rakefile")
-        Rake::Task[task].invoke(args)
+    def run(args)
+      if args.size > 0
+        if ["install", "examples", "jar"].include?(args[0])
+          load(TASKS_FILE)
+          Rake::Task[args.shift].invoke(*args)
+        elsif args.size == 2 && ["local"].include?(args[0]) && File.exist?(args[1])
+          load(TASKS_FILE)
+          Rake::Task['launch'].invoke(*args)
+        else
+          usage
+        end
       else
-        puts("\nUsage: redstorm install|examples|jar|topology_class_file_name")
-        exit(1)
+        usage
       end
     end
   end

data/lib/red_storm/simple_bolt.rb

@@ -0,0 +1,106 @@
+module RedStorm
+
+  class SimpleBolt
+    attr_reader :collector, :context, :config
+
+    # DSL class methods
+
+    def self.output_fields(*fields)
+      @fields = fields.map(&:to_s)
+    end
+
+    def self.on_receive(*args, &on_receive_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      method_name = args.first
+
+      self.receive_options.merge!(options)
+      @on_receive_block = block_given? ? on_receive_block : lambda {|tuple| self.send(method_name || :on_receive, tuple)}
+    end
+
+    def self.on_init(method_name = nil, &on_init_block)
+      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
+    end
+
+    def self.on_close(method_name = nil, &close_block)
+      @close_block = block_given? ? close_block : lambda {self.send(method_name || :on_close)}
+    end
+
+    # DSL instance methods
+
+    def unanchored_emit(*values)
+      @collector.emit(Values.new(*values)) 
+    end
+
+    def anchored_emit(tuple, *values)
+      @collector.emit(tuple, Values.new(*values)) 
+    end
+
+    def ack(tuple)
+      @collector.ack(tuple)
+    end
+
+    # Bolt proxy interface
+
+    def execute(tuple)
+      if (output = instance_exec(tuple, &self.class.on_receive_block)) && self.class.emit?
+        values_list = !output.is_a?(Array) ? [[output]] : !output.first.is_a?(Array) ? [output] : output
+        values_list.each{|values| self.class.anchor? ? anchored_emit(tuple, *values) : unanchored_emit(*values)}
+        @collector.ack(tuple) if self.class.ack?
+      end
+    end
+
+    def prepare(config, context, collector)
+      @collector = collector
+      @context = context
+      @config = config
+      instance_exec(&self.class.on_init_block)
+    end
+
+    def cleanup
+      instance_exec(&self.class.close_block)
+    end
+
+    def declare_output_fields(declarer)
+      declarer.declare(Fields.new(self.class.fields))
+    end
+
+    # default optional dsl methods/callbacks
+
+    def on_init; end
+    def on_close; end
+
+    private
+
+    def self.fields
+      @fields ||= []
+    end
+
+    def self.on_receive_block
+      @on_receive_block ||= lambda {|tuple| self.send(:on_receive, tuple)}
+    end
+
+    def self.on_init_block
+      @on_init_block ||= lambda {self.send(:on_init)}
+    end
+
+    def self.close_block
+      @close_block ||= lambda {self.send(:on_close)}
+    end
+
+    def self.receive_options
+      @receive_options ||= {:emit => true, :ack => false, :anchor => false}
+    end
+
+    def self.emit?
+      !!self.receive_options[:emit]
+    end
+
+    def self.ack?
+      !!self.receive_options[:ack]
+    end
+
+    def self.anchor?
+      !!self.receive_options[:anchor]
+    end
+  end
+end

data/lib/red_storm/simple_spout.rb

@@ -0,0 +1,136 @@
+module RedStorm
+
+  class SimpleSpout
+    attr_reader :config, :context, :collector
+
+    # DSL class methods
+
+    def self.set(options = {})
+      self.spout_options.merge!(options)
+    end
+
+    def self.output_fields(*fields)
+      @fields = fields.map(&:to_s)
+    end
+
+    def self.on_send(*args, &on_send_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      method_name = args.first
+      
+      self.send_options.merge!(options)
+      @on_send_block = block_given? ? on_send_block : lambda {self.send(method_name || :on_send)}
+    end
+
+    def self.on_init(method_name = nil, &on_init_block)
+      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
+    end
+
+    def self.on_close(method_name = nil, &on_close_block)
+      @on_close_block = block_given? ? on_close_block : lambda {self.send(method_name || :on_close)}
+    end
+
+    def self.on_ack(method_name = nil, &on_ack_block)
+      @on_ack_block = block_given? ? on_ack_block : lambda {|msg_id| self.send(method_name || :on_ack, msg_id)}
+    end
+
+    def self.on_fail(method_name = nil, &on_fail_block)
+      @on_fail_block = block_given? ? on_fail_block : lambda {|msg_id| self.send(method_name || :on_fail, msg_id)}
+    end
+
+    # DSL instance methods
+
+    def emit(*values)
+      @collector.emit(Values.new(*values)) 
+    end
+
+    # Spout proxy interface
+
+    def next_tuple
+      output = instance_exec(&self.class.on_send_block)
+      if self.class.emit?
+        if output
+          values = [output].flatten
+          @collector.emit(Values.new(*values))
+        else
+          sleep(0.1)
+        end
+      end
+    end
+
+    def open(config, context, collector)
+      @collector = collector
+      @context = context
+      @config = config
+      instance_exec(&self.class.on_init_block)
+    end
+
+    def close
+      instance_exec(&self.class.on_close_block)
+    end
+
+    def declare_output_fields(declarer)
+      declarer.declare(Fields.new(self.class.fields))
+    end
+
+    def is_distributed
+      self.class.is_distributed?
+    end
+
+    def ack(msg_id)
+      instance_exec(msg_id, &self.class.on_ack_block)
+    end
+
+    def fail(msg_id)
+      instance_exec(msg_id, &self.class.on_fail_block)
+    end
+
+    # default optional dsl methods/callbacks
+
+    def on_init; end
+    def on_close; end
+    def on_ack(msg_id); end
+    def on_fail(msg_id); end
+
+    private
+
+    def self.fields
+      @fields ||= []
+    end
+
+    def self.on_send_block
+      @on_send_block ||= lambda {self.send(:on_send)}
+    end
+
+    def self.on_init_block
+      @on_init_block ||= lambda {self.send(:on_init)}
+    end
+
+    def self.on_close_block
+      @on_close_block ||= lambda {self.send(:on_close)}
+    end
+
+    def self.on_ack_block
+      @on_ack_block ||= lambda {|msg_id| self.send(:on_ack, msg_id)}
+    end
+
+    def self.on_fail_block
+      @on_fail_block ||= lambda {|msg_id| self.send(:on_fail, msg_id)}
+    end
+
+    def self.send_options
+      @send_options ||= {:emit => true}
+    end
+
+    def self.spout_options
+      @spout_options ||= {:is_distributed => false}
+    end
+
+    def self.is_distributed?
+      !!self.spout_options[:is_distributed]
+    end
+
+    def self.emit?
+      !!self.send_options[:emit]
+    end
+  end
+end

data/lib/red_storm/simple_topology.rb

@@ -0,0 +1,191 @@
+module RedStorm
+
+  class SimpleTopology
+    attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
+
+    DEFAULT_SPOUT_PARALLELISM = 1
+    DEFAULT_BOLT_PARALLELISM = 1
+
+    class ComponentDefinition
+      attr_reader :clazz, :parallelism
+      attr_accessor :id
+
+      def initialize(component_class, id, parallelism)
+        @clazz = component_class
+        @id = id
+        @parallelism = parallelism
+      end
+    end
+
+    class SpoutDefinition < ComponentDefinition; end
+          
+    class BoltDefinition < ComponentDefinition
+      attr_accessor :sources
+
+      def initialize(*args)
+        super
+        @sources = []
+      end
+
+      def source(source_id, grouping)
+        @sources << [source_id.is_a?(Class) ? SimpleTopology.underscore(source_id) : source_id, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
+      end
+
+      def define_grouping(declarer)
+        @sources.each do |source_id, grouping|
+          grouper, params = grouping.first
+
+          case grouper
+          when :fields
+            declarer.fieldsGrouping(source_id, Fields.new(*params))
+          when :global
+            declarer.globalGrouping(source_id)
+          when :shuffle
+            declarer.shuffleGrouping(source_id)
+          when :none
+            declarer.noneGrouping(source_id)
+          when :all
+            declarer.allGrouping(source_id)
+          when :direct
+            declarer.directGrouping(source_id)
+          else
+            raise("unknown grouper=#{grouper.inspect}")
+          end
+        end
+      end
+    end
+
+    class Configurator
+      attr_reader :config
+
+      def initialize
+        @config = Config.new
+      end
+
+      def method_missing(sym, *args)
+        config_method = "set#{self.class.camel_case(sym)}"
+        @config.send(config_method, *args)
+      end
+
+      private
+
+      def self.camel_case(s)
+        s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      end
+    end
+
+    def self.spout(spout_class, options = {})
+      spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
+      spout = SpoutDefinition.new(spout_class, spout_options[:id], spout_options[:parallelism])
+      self.components << spout
+    end
+
+    def self.bolt(bolt_class, options = {}, &bolt_block)
+      bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+      bolt = BoltDefinition.new(bolt_class, bolt_options[:id], bolt_options[:parallelism])
+      bolt.instance_exec(&bolt_block)
+      self.components << bolt
+    end
+
+    def self.configure(name = nil, &configure_block)
+      @topology_name = name if name
+      @configure_block = configure_block if block_given?
+    end
+
+    def self.on_submit(method_name = nil, &submit_block)
+      @submit_block = block_given? ? submit_block : lambda {|env| self.send(method_name, env)}
+    end
+
+    # topology proxy interface
+
+    def start(base_class_path, env)
+      self.class.resolve_ids!(self.class.components)
+
+      builder = TopologyBuilder.new
+      self.class.spouts.each do |spout|
+        is_java = spout.clazz.name.split('::').first == 'Java'
+        builder.setSpout(spout.id, is_java ? spout.clazz.new : JRubySpout.new(base_class_path, spout.clazz.name), spout.parallelism)
+      end
+      self.class.bolts.each do |bolt|
+        is_java = bolt.clazz.name.split('::').first == 'Java'
+        declarer = builder.setBolt(bolt.id, is_java ? bolt.clazz.new : JRubyBolt.new(base_class_path, bolt.clazz.name), bolt.parallelism)
+        bolt.define_grouping(declarer)
+      end
+
+      configurator = Configurator.new
+      configurator.instance_exec(env, &self.class.configure_block)
+ 
+      case env
+      when :local
+        @cluster = LocalCluster.new
+        @cluster.submitTopology(self.class.topology_name, configurator.config, builder.createTopology)
+      when :cluster
+        StormSubmitter.submitTopology(self.class.topology_name, configurator.config, builder.createTopology);
+      else
+        raise("unsupported env=#{env.inspect}, expecting :local or :cluster")
+      end
+
+      instance_exec(env, &self.class.submit_block)
+    end
+
+    private
+
+    def self.resolve_ids!(components)
+      next_numeric_id = 1
+      resolved_names = {}
+
+      numeric_components, symbolic_components = components.partition{|c| c.id.is_a?(Fixnum)}
+      numeric_ids = numeric_components.map(&:id)
+
+      # assign numeric ids to symbolic ids
+      symbolic_components.each do |component|
+        id = component.id.to_s
+        raise("duplicate symbolic id in #{component.clazz.name} on id=#{id}") if resolved_names.has_key?(id)
+        next_numeric_id += 1 while numeric_ids.include?(next_numeric_id)
+        numeric_ids << next_numeric_id
+        resolved_names[id] = next_numeric_id
+      end
+
+      # reassign numeric ids to all components
+      components.each do |component|
+        unless component.id.is_a?(Fixnum)
+          component.id = resolved_names[component.id.to_s] || raise("cannot resolve #{component.clazz.name} id=#{component.id.to_s}")
+        end
+        if component.respond_to?(:sources)
+          component.sources.map! do |source_id, grouping|
+            id = source_id.is_a?(Fixnum) ? source_id : resolved_names[source_id.to_s] || raise("cannot resolve #{component.clazz.name} source id=#{source_id.to_s}")
+            [id, grouping]
+          end
+        end
+      end
+    end
+
+    def self.spouts
+      self.components.select{|c| c.is_a?(SpoutDefinition)}
+    end
+
+    def self.bolts
+      self.components.select{|c| c.is_a?(BoltDefinition)}
+    end
+
+    def self.components
+      @components ||= []
+    end
+
+    def self.topology_name
+      @topology_name ||= self.underscore(self.name)
+    end
+
+    def self.configure_block
+      @configure_block ||= lambda{|env|}
+    end
+
+    def self.submit_block
+      @submit_block ||= lambda{|env|}
+    end
+
+    def self.underscore(camel_case)
+      camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
+    end
+  end
+end