redstorm 0.6.5 → 0.6.6.beta1

data/examples/{simple → dsl}/random_sentence_spout.rb

@@ -2,7 +2,7 @@ require 'red_storm'
 
 module RedStorm
   module Examples
-    class RandomSentenceSpout < RedStorm::SimpleSpout
+    class RandomSentenceSpout < DSL::Spout
       output_fields :word
 
       on_send {@sentences[rand(@sentences.length)]}

data/examples/{simple → dsl}/redis_word_count_topology.rb

@@ -1,15 +1,15 @@
 require 'red_storm'
-require 'examples/simple/word_count_bolt'
+require 'examples/dsl/word_count_bolt'
 require 'redis'
 require 'thread'
 
 module RedStorm
   module Examples
 
-    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379
     # and emits each word items pop'ed from the queue.
 
-    class RedisWordSpout < RedStorm::SimpleSpout
+    class RedisWordSpout < DSL::Spout
       output_fields :word
 
       on_send {@q.pop.to_s if @q.size > 0}
@@ -18,7 +18,7 @@ module RedStorm
         @q = Queue.new
         @redis_reader = detach_redis_reader
       end
-      
+
       private
 
       def detach_redis_reader
@@ -35,16 +35,15 @@ module RedStorm
       end
     end
 
-    class RedisWordCountTopology < RedStorm::SimpleTopology
+    class RedisWordCountTopology < DSL::Topology
       spout RedisWordSpout
-          
+
       bolt WordCountBolt, :parallelism => 3 do
         source RedisWordSpout, :fields => ["word"]
       end
 
       configure do |env|
         debug true
-        # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3

data/examples/{simple → dsl}/ruby_version_topology.rb

@@ -1,30 +1,30 @@
+require 'java'
 require 'red_storm'
+java_import 'java.lang.System'
+
 
 # this example topology only prints the Ruby version string. No tuple is emitted.
 
 module RedStorm
   module Examples
-    class VersionSpout < RedStorm::SimpleSpout
+    class VersionSpout < DSL::Spout
       output_fields :dummy
       on_init do
+        log.info("***************** REDSTORM VERSION=#{VERSION}")
         log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")
         log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}")
-        log.info("***************** VERSION=#{VERSION}")
         log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}")
         log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}")
+        log.info("***************** JAVA VERSION=#{System.properties["java.runtime.version"]}")
       end
       on_send {}
     end
 
-    class RubyVersionTopology < RedStorm::SimpleTopology
+    class RubyVersionTopology < DSL::Topology
       spout VersionSpout
-            
-      configure do |env|
-        debug true
 
-        # force the JRuby version property for this topology. this will only affect remote cluster execution
-        # for local execution use the --1.8|--1.9 switch when launching
-        # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+      configure do |env|
+        debug false
       end
 
       on_submit do |env|

data/examples/{simple → dsl}/split_sentence_bolt.rb

@@ -2,31 +2,31 @@ require 'red_storm'
 
 module RedStorm
   module Examples
-    class SplitSentenceBolt < RedStorm::SimpleBolt
+    class SplitSentenceBolt < DSL::Bolt
       output_fields :word
 
       # block declaration style using auto-emit (default)
       #
-      on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}}
+      on_receive {|tuple| tuple[0].split(' ').map{|w| [w]}}
 
       # block declaration style no auto-emit
       #
-      # on_receive :emit => false do |tuple| 
-      #   tuple.getString(0).split(' ').each{|w| unanchored_emit(w)}
+      # on_receive :emit => false do |tuple|
+      #   tuple[0].split(' ').each{|w| unanchored_emit(w)}
       # end
 
       # alternate declaration style using on_receive method
       #
       # on_receive :emit => true
       # def on_receive(tuple)
-      #   tuple.getString(0).split(' ').map{|w| [w]}
+      #   tuple[0].split(' ').map{|w| [w]}
       # end
 
       # alternate declaration style using any specific method
       #
       # on_receive :my_method, :emit => true
       # def my_method(tuple)
-      #   tuple.getString(0).split(' ').map{|w| [w]}
+      #   tuple[0].split(' ').map{|w| [w]}
       # end
     end
   end

data/examples/{simple → dsl}/word_count_bolt.rb

@@ -2,14 +2,14 @@ require 'red_storm'
 
 module RedStorm
   module Examples
-    class WordCountBolt < RedStorm::SimpleBolt
+    class WordCountBolt < DSL::Bolt
       output_fields :word, :count
       on_init {@counts = Hash.new{|h, k| h[k] = 0}}
 
       # block declaration style using auto-emit (default)
       #
       on_receive do |tuple|
-        word = tuple.getValue(0).to_s
+        word = tuple[0].to_s
         @counts[word] += 1
 
         [word, @counts[word]]

data/examples/{simple → dsl}/word_count_topology.rb

@@ -1,17 +1,17 @@
 require 'red_storm'
-require 'examples/simple/random_sentence_spout'
-require 'examples/simple/split_sentence_bolt'
-require 'examples/simple/word_count_bolt'
+require 'examples/dsl/random_sentence_spout'
+require 'examples/dsl/split_sentence_bolt'
+require 'examples/dsl/word_count_bolt'
 
 module RedStorm
   module Examples
-    class WordCountTopology < SimpleTopology
+    class WordCountTopology < DSL::Topology
       spout RandomSentenceSpout, :parallelism => 2
-      
+
       bolt SplitSentenceBolt, :parallelism => 2 do
         source RandomSentenceSpout, :shuffle
       end
-      
+
       bolt WordCountBolt, :parallelism => 2 do
         source SplitSentenceBolt, :fields => ["word"]
       end

data/examples/shell/shell_topology.rb

@@ -3,7 +3,7 @@ require 'thread'
 
 java_import 'redstorm.storm.jruby.JRubyShellBolt'
 
-class SimpleSpout < RedStorm::SimpleSpout
+class SimpleSpout < RedStorm::DSL::Spout
   on_init do
     @q = Queue.new
     @q << "the quick red fox"
@@ -16,7 +16,7 @@ class SimpleSpout < RedStorm::SimpleSpout
   end
 end
 
-class ShellTopology < RedStorm::SimpleTopology
+class ShellTopology < RedStorm::DSL::Topology
   spout SimpleSpout do
     output_fields :string
   end

data/ivy/storm_dependencies.xml

@@ -1,8 +1,8 @@
 <?xml version="1.0"?>
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="redstorm" module="storm-deps"/>
   <dependencies>
-    <dependency org="storm" name="storm" rev="0.8.2" conf="default" transitive="true" />
+    <dependency org="storm" name="storm" rev="0.9.0-wip16" conf="default" transitive="true" />
     <override org="org.slf4j" module="slf4j-log4j12" rev="1.6.3"/>
   </dependencies>
 </ivy-module>

data/ivy/topology_dependencies.xml

@@ -1,7 +1,15 @@
 <?xml version="1.0"?>
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="redstorm" module="topology-deps"/>
   <dependencies>
-    <dependency org="org.jruby" name="jruby-core" rev="1.7.3" conf="default" transitive="true"/>
+    <dependency org="org.jruby" name="jruby-core" rev="1.7.4" conf="default" transitive="true"/>
+
+    <!-- explicitely specify jffi to also fetch the native jar. make sure to update jffi version matching jruby-core version -->
+    <!-- this is the only way I found using Ivy to fetch the native jar -->
+    <dependency org="com.github.jnr" name="jffi" rev="1.2.5" conf="default" transitive="true">
+      <artifact name="jffi" type="jar" />
+      <artifact name="jffi" type="jar" m:classifier="native"/>
+    </dependency>
+
   </dependencies>
 </ivy-module>

data/lib/red_storm.rb

@@ -1,9 +1,10 @@
 require 'rubygems'
 
-require 'red_storm/environment'
 require 'red_storm/version'
+require 'red_storm/environment'
 require 'red_storm/configuration'
-require 'red_storm/simple_bolt'
-require 'red_storm/simple_spout'
-require 'red_storm/simple_topology'
-require 'red_storm/simple_drpc_topology'
+require 'red_storm/dsl/bolt'
+require 'red_storm/dsl/spout'
+require 'red_storm/dsl/topology'
+require 'red_storm/dsl/drpc_topology'
+require 'red_storm/dsl/tuple'

data/lib/red_storm/application.rb

@@ -29,8 +29,8 @@ CUSTOM_IVY_TOPOLOGY_DEPENDENCIES = "#{DST_IVY_DIR}/topology_dependencies.xml"
 
 module RedStorm
 
-  class Application 
-    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
+  class Application
+    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake"
 
     def self.local_storm_command(class_file, ruby_mode = nil)
       src_dir = File.expand_path(File.dirname(class_file))
@@ -43,9 +43,9 @@ module RedStorm
 
     def self.usage
       puts("usage: redstorm version")
-      puts("       redstorm install")
+      puts("       redstorm install [--JVM_VERSION] (ex.: --1.6 or --1.7) default is current JVM version")
       puts("       redstorm deps")
-      puts("       redstorm build")
+      puts("       redstorm build [--JVM_VERSION] (ex.: --1.6 or --1.7) default is current JVM version")
       puts("       redstorm examples")
       puts("       redstorm bundle [BUNDLER_GROUP]")
       puts("       redstorm jar DIR1, [DIR2, ...]")
@@ -59,7 +59,7 @@ module RedStorm
         if args[0] == "version"
           puts("RedStorm v#{VERSION}")
           exit
-        elsif ["install", "examples", "jar", "bundle", "deps", "build"].include?(args[0])
+        elsif ["examples", "jar", "bundle", "deps", "install", "build"].include?(args[0])
           load(TASKS_FILE)
           Rake::Task[args.shift].invoke(args.join(":"))
           exit

data/lib/red_storm/dsl/bolt.rb

@@ -0,0 +1,155 @@
+require 'java'
+require 'red_storm/configurator'
+require 'red_storm/environment'
+require 'pathname'
+
+module RedStorm
+  module DSL
+
+    class BoltError < StandardError; end
+
+    class Bolt
+      attr_reader :collector, :context, :config
+
+      # DSL class methods
+
+      def self.log
+        @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
+      end
+
+      def self.output_fields(*fields)
+        @fields = fields.map(&:to_s)
+      end
+
+      def self.configure(&configure_block)
+        @configure_block = block_given? ? configure_block : lambda {}
+      end
+
+      def self.on_receive(*args, &on_receive_block)
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        method_name = args.first
+
+        self.receive_options.merge!(options)
+
+        # indirecting through a lambda defers the method lookup at invocation time
+        # and the performance penalty is negligible
+        body = block_given? ? on_receive_block : lambda{|tuple| self.send((method_name || :on_receive).to_sym, tuple)}
+        define_method(:on_receive, body)
+      end
+
+      def self.on_init(method_name = nil, &on_init_block)
+        body = block_given? ? on_init_block : lambda {self.send((method_name || :on_init).to_sym)}
+        define_method(:on_init, body)
+      end
+
+      def self.on_close(method_name = nil, &on_close_block)
+        body = block_given? ? on_close_block : lambda {self.send((method_name || :on_close).to_sym)}
+        define_method(:on_close, body)
+      end
+
+      # DSL instance methods
+
+      def log
+        self.class.log
+      end
+
+      def unanchored_emit(*values)
+        @collector.emit(Values.new(*values))
+      end
+
+      def anchored_emit(tuple, *values)
+        @collector.emit(tuple, Values.new(*values))
+      end
+
+      def ack(tuple)
+        @collector.ack(tuple)
+      end
+
+      def fail(tuple)
+        @collector.fail(tuple)
+      end
+
+      # Bolt proxy interface
+
+      def execute(tuple)
+        output = on_receive(tuple)
+        if output && self.class.emit?
+          values_list = !output.is_a?(Array) ? [[output]] : !output.first.is_a?(Array) ? [output] : output
+          values_list.each{|values| self.class.anchor? ? anchored_emit(tuple, *values) : unanchored_emit(*values)}
+          @collector.ack(tuple) if self.class.ack?
+        end
+      end
+
+      def prepare(config, context, collector)
+        @collector = collector
+        @context = context
+        @config = config
+
+        on_init
+      end
+
+      def cleanup
+        on_close
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new(self.class.fields))
+      end
+
+      def get_component_configuration
+        configurator = Configurator.new
+        configurator.instance_exec(&self.class.configure_block)
+        configurator.config
+      end
+
+      private
+
+      # default noop optional dsl callbacks
+      def on_init; end
+      def on_close; end
+
+      def self.fields
+        @fields ||= []
+      end
+
+      def self.configure_block
+        @configure_block ||= lambda {}
+      end
+
+      def self.receive_options
+        @receive_options ||= {:emit => true, :ack => false, :anchor => false}
+      end
+
+      def self.emit?
+        !!self.receive_options[:emit]
+      end
+
+      def self.ack?
+        !!self.receive_options[:ack]
+      end
+
+      def self.anchor?
+        !!self.receive_options[:anchor]
+      end
+
+      # below non-dry see Spout class
+      def self.inherited(subclass)
+        path = (caller.first.to_s =~ /^(.+):\d+.*$/) ? $1 : raise(BoltError, "unable to extract base topology class path from #{caller.first.inspect}")
+        subclass.base_class_path = Pathname.new(path).relative_path_from(Pathname.new(RedStorm::BASE_PATH)).to_s
+      end
+
+      def self.base_class_path=(path)
+        @base_class_path = path
+      end
+
+      def self.base_class_path
+        @base_class_path
+      end
+
+    end
+  end
+
+  # for backward compatibility
+  SimpleBolt = DSL::Bolt
+
+end

data/lib/red_storm/dsl/drpc_topology.rb

@@ -0,0 +1,92 @@
+require 'java'
+require 'red_storm/configuration'
+require 'red_storm/configurator'
+
+module RedStorm
+  module DSL
+
+    class InputBoltDefinition < Topology::BoltDefinition
+      attr_accessor :grouping
+
+      def initialize(*args)
+        super
+        @grouping = :none
+      end
+
+      def grouping(grouping)
+        @grouping = grouping
+      end
+
+      def define_grouping(declarer)
+        case @grouping
+        when :fields
+          declarer.fieldsGrouping(Fields.new(*([params].flatten.map(&:to_s))))
+        when :global
+          declarer.globalGrouping()
+        when :shuffle
+          declarer.shuffleGrouping()
+        when :local_or_shuffle
+          declarer.localOrShuffleGrouping()
+        when :none
+          declarer.noneGrouping()
+        when :all
+          declarer.allGrouping()
+        when :direct
+          declarer.directGrouping()
+        else
+          raise("unknown grouper=#{grouper.inspect}")
+        end
+      end
+    end
+
+    class DRPCTopology < Topology
+
+      def self.spout
+        raise TopologyDefinitionError, "DRPC spout is already defined"
+      end
+
+      def start(base_class_path, env)
+        builder = Java::BacktypeStormDrpc::LinearDRPCTopologyBuilder.new(self.class.topology_name)
+
+        self.class.bolts.each do |bolt|
+          declarer = builder.addBolt(bolt.new_instance(base_class_path), bolt.parallelism.to_java)
+          declarer.addConfigurations(bolt.config)
+          bolt.define_grouping(declarer)
+        end
+
+        # set the JRuby compatibility mode option for Storm workers, default to current JRuby mode
+        defaults = {"topology.worker.childopts" => "-Djruby.compat.version=#{RedStorm.jruby_mode_token}"}
+
+        configurator = Configurator.new(defaults)
+        configurator.instance_exec(env, &self.class.configure_block)
+
+        drpc = nil
+        if env == :local
+          drpc = LocalDRPC.new
+          submitter = @cluster = LocalCluster.new
+          submitter.submitTopology(self.class.topology_name, configurator.config, builder.createLocalTopology(drpc))
+        else
+          submitter = StormSubmitter
+          submitter.submitTopology(self.class.topology_name, configurator.config, builder.createRemoteTopology)
+        end
+        instance_exec(env, drpc, &self.class.submit_block)
+      end
+
+      def self.input_bolt(bolt_class, *args, &bolt_block)
+        set_topology_class!
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        contructor_args = !args.empty? ? args.pop : []
+        bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
+
+        bolt = InputBoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
+        raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
+        bolt.instance_exec(&bolt_block)
+        self.components << bolt
+      end
+    end
+  end
+
+  # for backward compatibility
+  SimpleDRPCTopology = DSL::DRPCTopology
+
+end