redstorm 0.6.4 → 0.6.5

data/examples/native/cluster_word_count_topology.rb

@@ -10,16 +10,16 @@ module RedStorm
 
       def start(base_class_path, env)
         builder = TopologyBuilder.new
-        builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
-        builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 4).shuffleGrouping('RandomSentenceSpout')
-        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+        builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
+        builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
 
         conf = Backtype::Config.new
         conf.setDebug(true)
-        conf.setNumWorkers(20);
+        conf.setNumWorkers(4);
         conf.setMaxSpoutPending(1000);
         StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
       end
     end
   end
-end
+end

data/examples/native/local_exclamation_topology.rb

@@ -1,6 +1,6 @@
 java_import 'backtype.storm.testing.TestWordSpout'
 
-require 'lib/red_storm'
+require 'red_storm'
 require 'examples/native/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
@@ -12,14 +12,14 @@ module RedStorm
 
       def start(base_class_path, env)
         builder = TopologyBuilder.new
-        
-        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
-        builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('TestWordSpout')
-        builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('ExclamationBolt1')
-                
+
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 1)
+        builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt', []), 2).shuffleGrouping('ExclamationBolt1')
+
         conf = Backtype::Config.new
         conf.setDebug(true)
-        
+
         cluster = LocalCluster.new
         cluster.submitTopology("exclamation", conf, builder.createTopology)
         sleep(5)
@@ -28,4 +28,4 @@ module RedStorm
       end
     end
   end
-end
+end

data/examples/native/local_exclamation_topology2.rb

@@ -1,6 +1,6 @@
 java_import 'backtype.storm.testing.TestWordSpout'
 
-require 'lib/red_storm'
+require 'red_storm'
 
 module RedStorm
   module Examples
@@ -29,14 +29,14 @@ module RedStorm
 
       def start(base_class_path, env)
         builder = TopologyBuilder.new
-        
-        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
-        builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 3).shuffleGrouping('TestWordSpout')
-        builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 2).shuffleGrouping('ExclamationBolt21')
-                
+
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 2)
+        builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2", []), 2).shuffleGrouping('ExclamationBolt21')
+
         conf = Backtype::Config.new
         conf.setDebug(true)
-        
+
         cluster = LocalCluster.new
         cluster.submitTopology("exclamation", conf, builder.createTopology)
         sleep(5)

data/examples/native/local_redis_word_count_topology.rb

@@ -1,12 +1,11 @@
-require 'bundler/setup'
+require 'red_storm'
+require 'examples/native/word_count_bolt'
 require 'redis'
 require 'thread'
-require 'lib/red_storm'
-require 'examples/native/word_count_bolt'
 
 module RedStorm
   module Examples
-    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379
     # and emits each word items pop'ed from the queue.
     class RedisWordSpout
       def open(conf, context, collector)
@@ -14,7 +13,7 @@ module RedStorm
         @q = Queue.new
         @redis_reader = detach_redis_reader
       end
-      
+
       def next_tuple
         # per doc nextTuple should not block, and sleep a bit when there's no data to process.
         if @q.size > 0
@@ -52,8 +51,8 @@ module RedStorm
 
       def start(base_class_path, env)
         builder = TopologyBuilder.new
-        builder.setSpout('RedisWordSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RedisWordSpout"), 1)
-        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 3).fieldsGrouping('RedisWordSpout', Fields.new("word"))
+        builder.setSpout('RedisWordSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RedisWordSpout", []), 1)
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 3).fieldsGrouping('RedisWordSpout', Fields.new("word"))
 
         conf = Backtype::Config.new
         conf.setDebug(true)
@@ -66,4 +65,4 @@ module RedStorm
       end
     end
   end
-end
+end

data/examples/native/local_word_count_topology.rb

@@ -1,4 +1,4 @@
-require 'lib/red_storm'
+require 'red_storm'
 require 'examples/native/random_sentence_spout'
 require 'examples/native/split_sentence_bolt'
 require 'examples/native/word_count_bolt'
@@ -10,9 +10,9 @@ module Examples
 
     def start(base_class_path, env)
       builder = TopologyBuilder.new
-      builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
-      builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 8).shuffleGrouping('RandomSentenceSpout')
-      builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+      builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout", []), 1)
+      builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt", []), 2).shuffleGrouping('RandomSentenceSpout')
+      builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt", []), 2).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
 
       conf = Backtype::Config.new
       conf.setDebug(true)
@@ -24,4 +24,4 @@ module Examples
       cluster.shutdown
     end
   end
-end
+end

data/examples/simple/exclamation_topology.rb

@@ -1,36 +1,32 @@
 java_import 'backtype.storm.testing.TestWordSpout'
 
+require 'red_storm'
 require 'examples/simple/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
 module RedStorm
   module Examples
-    class ExclamationTopology < RedStorm::SimpleTopology
-      spout TestWordSpout, :parallelism => 5 do
+    class ExclamationTopology < SimpleTopology
+      spout TestWordSpout, :parallelism => 2 do
         debug true
       end
       
       bolt ExclamationBolt, :parallelism => 2 do
         source TestWordSpout, :shuffle
-        # max_task_parallelism 1
       end
       
       bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
         source ExclamationBolt, :shuffle
-        # max_task_parallelism 1
         debug true
       end
 
       configure do |env|
         debug false
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 40
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 4
+          max_spout_pending(1000)
         end
       end
 

data/examples/simple/exclamation_topology2.rb

@@ -1,9 +1,10 @@
-java_import 'backtype.storm.testing.TestWordSpout'
-require 'red_storm'
-
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 # and a locally defined ExclamationBolt
 
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'red_storm'
+
 module RedStorm
   module Examples
     class ExclamationBolt < RedStorm::SimpleBolt
@@ -12,9 +13,9 @@ module RedStorm
     end
 
     class ExclamationTopology2 < RedStorm::SimpleTopology
-      spout TestWordSpout, :parallelism => 10
+      spout TestWordSpout, :parallelism => 2
       
-      bolt ExclamationBolt, :parallelism => 3 do
+      bolt ExclamationBolt, :parallelism => 2 do
         source TestWordSpout, :shuffle
       end
       
@@ -24,13 +25,10 @@ module RedStorm
 
       configure do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 3
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 4
+          max_spout_pending(1000)
         end
       end
 

data/examples/simple/hello_world_topology.rb

@@ -0,0 +1,22 @@
+require 'red_storm'
+
+class HelloWorldSpout < RedStorm::SimpleSpout
+  on_init {@words = ["hello", "world"]}
+  on_send {@words.shift unless @words.empty?}
+end
+
+class HelloWorldBolt < RedStorm::SimpleBolt
+  on_receive :emit => false do |tuple|
+    log.info(tuple.getString(0))
+  end
+end
+
+class HelloWorldTopology < RedStorm::SimpleTopology
+  spout HelloWorldSpout do
+    output_fields :word
+  end
+
+  bolt HelloWorldBolt do
+    source HelloWorldSpout, :global
+  end
+end

data/examples/simple/kafka_topology.rb

@@ -1,9 +1,10 @@
-require 'red_storm'
 java_import 'storm.kafka.KafkaConfig'
 java_import 'storm.kafka.SpoutConfig'
 java_import 'storm.kafka.StringScheme'
 java_import 'storm.kafka.KafkaSpout'
 
+require 'red_storm'
+
 # the KafkaTopology obviously requires a Kafka server running, you can ajust the
 # host and port below. 
 #

data/examples/simple/redis_word_count_topology.rb

@@ -1,9 +1,7 @@
-require 'rubygems'
 require 'red_storm'
-
+require 'examples/simple/word_count_bolt'
 require 'redis'
 require 'thread'
-require 'examples/simple/word_count_bolt'
 
 module RedStorm
   module Examples
@@ -46,14 +44,14 @@ module RedStorm
 
       configure do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3
         when :cluster
           max_task_parallelism 5
           num_workers 20
-          max_spout_pending(1000);
+          max_spout_pending(1000)
         end
       end
     end

data/examples/simple/ruby_version_topology.rb

@@ -6,7 +6,13 @@ module RedStorm
   module Examples
     class VersionSpout < RedStorm::SimpleSpout
       output_fields :dummy
-      on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")}
+      on_init do
+        log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")
+        log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}")
+        log.info("***************** VERSION=#{VERSION}")
+        log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}")
+        log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}")
+      end
       on_send {}
     end
 

data/examples/simple/word_count_topology.rb

@@ -1,3 +1,4 @@
+require 'red_storm'
 require 'examples/simple/random_sentence_spout'
 require 'examples/simple/split_sentence_bolt'
 require 'examples/simple/word_count_bolt'
@@ -5,25 +6,22 @@ require 'examples/simple/word_count_bolt'
 module RedStorm
   module Examples
     class WordCountTopology < SimpleTopology
-      spout RandomSentenceSpout, :parallelism => 5
+      spout RandomSentenceSpout, :parallelism => 2
       
-      bolt SplitSentenceBolt, :parallelism => 8 do
+      bolt SplitSentenceBolt, :parallelism => 2 do
         source RandomSentenceSpout, :shuffle
       end
       
-      bolt WordCountBolt, :parallelism => 12 do
+      bolt WordCountBolt, :parallelism => 2 do
         source SplitSentenceBolt, :fields => ["word"]
       end
 
       configure :word_count do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 3
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 6
+          max_spout_pending(1000)
         end
       end
 

data/ivy/settings.xml

@@ -1,3 +1,4 @@
+<?xml version="1.0"?>
 <ivysettings>
   <settings defaultResolver="repositories"/>
   <resolvers>

data/ivy/storm_dependencies.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<ivy-module version="2.0">
+  <info organisation="redstorm" module="storm-deps"/>
+  <dependencies>
+    <dependency org="storm" name="storm" rev="0.8.2" conf="default" transitive="true" />
+    <override org="org.slf4j" module="slf4j-log4j12" rev="1.6.3"/>
+  </dependencies>
+</ivy-module>

data/ivy/topology_dependencies.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<ivy-module version="2.0">
+  <info organisation="redstorm" module="topology-deps"/>
+  <dependencies>
+    <dependency org="org.jruby" name="jruby-core" rev="1.7.3" conf="default" transitive="true"/>
+  </dependencies>
+</ivy-module>

data/lib/red_storm.rb

@@ -6,3 +6,4 @@ require 'red_storm/configuration'
 require 'red_storm/simple_bolt'
 require 'red_storm/simple_spout'
 require 'red_storm/simple_topology'
+require 'red_storm/simple_drpc_topology'

data/lib/red_storm/application.rb

@@ -7,7 +7,7 @@ TARGET_LIB_DIR = "#{TARGET_DIR}/lib"
 TARGET_SRC_DIR = "#{TARGET_DIR}/src"
 TARGET_GEM_DIR = "#{TARGET_DIR}/gems/gems"
 TARGET_SPECS_DIR = "#{TARGET_DIR}/gems/specifications"
-TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"  
+TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"
 TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency"
 TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked"
 TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar"
@@ -20,13 +20,15 @@ DST_EXAMPLES = "#{CWD}/examples"
 
 SRC_IVY_DIR = "#{RedStorm::REDSTORM_HOME}/ivy"
 DST_IVY_DIR = "#{CWD}/ivy"
-CUSTOM_DEPENDENCIES = "#{CWD}/Dependencies"
 DEFAULT_IVY_SETTINGS = "#{SRC_IVY_DIR}/settings.xml"
 CUSTOM_IVY_SETTINGS = "#{DST_IVY_DIR}/settings.xml"
-
+DEFAULT_IVY_STORM_DEPENDENCIES = "#{SRC_IVY_DIR}/storm_dependencies.xml"
+CUSTOM_IVY_STORM_DEPENDENCIES = "#{DST_IVY_DIR}/storm_dependencies.xml"
+DEFAULT_IVY_TOPOLOGY_DEPENDENCIES = "#{SRC_IVY_DIR}/topology_dependencies.xml"
+CUSTOM_IVY_TOPOLOGY_DEPENDENCIES = "#{DST_IVY_DIR}/topology_dependencies.xml"
 
 module RedStorm
-  
+
   class Application 
     TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
 
@@ -38,7 +40,7 @@ module RedStorm
     def self.cluster_storm_command(class_file, ruby_mode = nil)
       "storm jar #{TARGET_CLUSTER_JAR} -Djruby.compat.version=#{RedStorm.jruby_mode_token(ruby_mode)} redstorm.TopologyLauncher cluster #{class_file}"
     end
-    
+
     def self.usage
       puts("usage: redstorm version")
       puts("       redstorm install")
@@ -76,10 +78,10 @@ module RedStorm
     end
 
     def self.subshell(command)
-      out = IO.popen(command, {:err => [:child, :out]}) {|io| io.read}
+      out = IO.popen(command, STDERR => STDOUT) {|io| io.read}
       [!!$?.success?, out]
     end
 
   end
 
-end
+end

data/lib/red_storm/configurator.rb

@@ -20,7 +20,7 @@ module RedStorm
     private
 
     def self.camel_case(s)
-      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_|\.)(.)/) { $1.upcase }
     end
   end
 end