redstorm 0.2.1 → 0.4.0

data/CHANGELOG.md

@@ -12,4 +12,11 @@
 - more doc in README
 
 # 0.2.1, 11-23-2011
-- gems support in production cluster
+- gems support in production cluster
+
+# 0.3.0, 12-08-2011
+- Storm 0.6.0
+
+# 0.4.0, 02-08-2012
+- Storm 0.6.2
+- JRuby 1.6.6

data/README.md

@@ -1,17 +1,10 @@
-# RedStorm v0.2.1 - JRuby on Storm
+# RedStorm v0.4.0 - JRuby on Storm
 
 RedStorm provides the JRuby integration for the [Storm][storm] distributed realtime computation system.
 
-## Changes from 0.1.x
-
-- this release introduces the *simple* DSL. Topology, Spout and Bolt classes can inherit from the SimpleTopology, SimpleSpout and SimpleBolt classes which provides a very clean and consise DSL. See [examples/simple](https://github.com/colinsurprenant/redstorm/tree/master/examples/simple).
-- use the same SimpleTopology class for local development cluster or remote production cluster.
-- the `redstorm` command has a new syntax.
-- gems support in production cluster
-
 ## Dependencies
 
-This has been tested on OSX 10.6.8 and Linux 10.04 using Storm 0.5.4 and JRuby 1.6.5
+This has been tested on OSX 10.6.8 and Linux 10.04 using Storm 0.6.2 and JRuby 1.6.6
 
 ## Installation
 ``` sh
@@ -241,7 +234,7 @@ configure topology_name do |env|
 end
 ```
 
-The `configure` statement is **optional**.
+The `configure` statement is **required**.
 
 - `topology_name` — alternate topology name (**default** is topology class name)
 - `env` — is set to `:local` or `:cluster` for you to set enviroment specific configurations
@@ -463,10 +456,10 @@ end
 
 ### Requirements
 
-- JRuby 1.6.5
-- rake gem
-- ruby-maven gem
-- rspec gem
+- JRuby 1.6.6
+- rake gem ~> 0.9.2.2
+- ruby-maven gem ~> 3.0.3.0.28.5
+- rspec gem ~> 2.8.0
 
 ### Contribute
 

data/examples/native/cluster_word_count_topology.rb

@@ -1,18 +1,25 @@
+require 'red_storm'
 require 'examples/native/random_sentence_spout'
 require 'examples/native/split_sentence_bolt'
 require 'examples/native/word_count_bolt'
 
-class ClusterWordCountTopology
-  def start(base_class_path, env)
-    builder = TopologyBuilder.new
-    builder.setSpout(1, JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
-    builder.setBolt(2, JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 4).shuffleGrouping(1)
-    builder.setBolt(3, JRubyBolt.new(base_class_path, "WordCountBolt"), 4).fieldsGrouping(2, Fields.new("word"))
+module RedStorm
+  module Examples
+    class ClusterWordCountTopology
+      RedStorm::Configuration.topology_class = self
 
-    conf = Config.new
-    conf.setDebug(true)
-    conf.setNumWorkers(20);
-    conf.setMaxSpoutPending(1000);
-    StormSubmitter.submitTopology("word-count", conf, builder.createTopology);
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
+        builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 4).shuffleGrouping('RandomSentenceSpout')
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+
+        conf = Config.new
+        conf.setDebug(true)
+        conf.setNumWorkers(20);
+        conf.setMaxSpoutPending(1000);
+        StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
+      end
+    end
   end
 end

data/examples/native/exclamation_bolt.rb

@@ -1,14 +1,18 @@
-class ExclamationBolt
-  def prepare(conf, context, collector)
-    @collector = collector
-  end
+module RedStorm
+  module Examples
+    class ExclamationBolt
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
 
-  def execute(tuple)
-    @collector.emit(tuple, Values.new(tuple.getString(0) + "!!!"))
-    @collector.ack(tuple)
-  end
+      def execute(tuple)
+        @collector.emit(tuple, Values.new(tuple.getString(0) + "!!!"))
+        @collector.ack(tuple)
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word"))
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
   end
 end

data/examples/native/local_exclamation_topology.rb

@@ -1,23 +1,31 @@
 java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'lib/red_storm'
 require 'examples/native/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
-class LocalExclamationTopology
-  def start(base_class_path, env)
-    builder = TopologyBuilder.new
-    
-    builder.setSpout(1, TestWordSpout.new, 10)     
-    builder.setBolt(2, JRubyBolt.new(base_class_path, "ExclamationBolt"), 3).shuffleGrouping(1)
-    builder.setBolt(3, JRubyBolt.new(base_class_path, "ExclamationBolt"), 2).shuffleGrouping(2)
-            
-    conf = Config.new
-    conf.setDebug(true)
-    
-    cluster = LocalCluster.new
-    cluster.submitTopology("test", conf, builder.createTopology)
-    sleep(5)
-    cluster.killTopology("test")
-    cluster.shutdown
+module RedStorm
+  module Examples
+    class LocalExclamationTopology
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
+        builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('ExclamationBolt1')
+                
+        conf = Config.new
+        conf.setDebug(true)
+        
+        cluster = LocalCluster.new
+        cluster.submitTopology("exclamation", conf, builder.createTopology)
+        sleep(5)
+        cluster.killTopology("exclamation")
+        cluster.shutdown
+      end
+    end
   end
-end
+end

data/examples/native/local_exclamation_topology2.rb

@@ -1,37 +1,45 @@
 java_import 'backtype.storm.testing.TestWordSpout'
 
-class ExclamationBolt2
-  def prepare(conf, context, collector)
-    @collector = collector
-  end
+require 'lib/red_storm'
 
-  def execute(tuple)
-    @collector.emit(tuple, Values.new(tuple.getString(0) + "!!!"))
-    @collector.ack(tuple)
-  end
+module RedStorm
+  module Examples
+    class ExclamationBolt2
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word"))
-  end
-end
+      def execute(tuple)
+        @collector.emit(tuple, Values.new("!#{tuple.getString(0)}!"))
+        @collector.ack(tuple)
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
+
+    # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
-# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+    class LocalExclamationTopology2
+      RedStorm::Configuration.topology_class = self
 
-class LocalExclamationTopology2
-  def start(base_class_path, env)
-    builder = TopologyBuilder.new
-    
-    builder.setSpout(1, TestWordSpout.new, 10)     
-    builder.setBolt(2, JRubyBolt.new(base_class_path, "ExclamationBolt2"), 3).shuffleGrouping(1)
-    builder.setBolt(3, JRubyBolt.new(base_class_path, "ExclamationBolt2"), 2).shuffleGrouping(2)
-            
-    conf = Config.new
-    conf.setDebug(true)
-    
-    cluster = LocalCluster.new
-    cluster.submitTopology("test", conf, builder.createTopology)
-    sleep(5)
-    cluster.killTopology("test")
-    cluster.shutdown
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
+        builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 3).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 2).shuffleGrouping('ExclamationBolt21')
+                
+        conf = Config.new
+        conf.setDebug(true)
+        
+        cluster = LocalCluster.new
+        cluster.submitTopology("exclamation", conf, builder.createTopology)
+        sleep(5)
+        cluster.killTopology("exclamation")
+        cluster.shutdown
+      end
+    end
   end
 end

data/examples/native/local_redis_word_count_topology.rb

@@ -1,58 +1,65 @@
 require 'redis'
 require 'thread'
+require 'lib/red_storm'
 require 'examples/native/word_count_bolt'
 
-# RedisWordSpout reads the Redis queue "test" on localhost:6379 
-# and emits each word items pop'ed from the queue.
-class RedisWordSpout
-  def open(conf, context, collector)
-    @collector = collector
-    @q = Queue.new
-    @redis_reader = detach_redis_reader
-  end
-  
-  def next_tuple
-    # per doc nextTuple should not block, and sleep a bit when there's no data to process.
-    if @q.size > 0
-      @collector.emit(Values.new(@q.pop))
-    else
-      sleep(0.1)
-    end
-  end
+module RedStorm
+  module Examples
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # and emits each word items pop'ed from the queue.
+    class RedisWordSpout
+      def open(conf, context, collector)
+        @collector = collector
+        @q = Queue.new
+        @redis_reader = detach_redis_reader
+      end
+      
+      def next_tuple
+        # per doc nextTuple should not block, and sleep a bit when there's no data to process.
+        if @q.size > 0
+          @collector.emit(Values.new(@q.pop))
+        else
+          sleep(0.1)
+        end
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word"))
-  end
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
 
-  private
+      private
 
-  def detach_redis_reader
-    Thread.new do
-      Thread.current.abort_on_exception = true
+      def detach_redis_reader
+        Thread.new do
+          Thread.current.abort_on_exception = true
 
-      redis = Redis.new(:host => "localhost", :port => 6379)
-      loop do
-        if data = redis.blpop("test", 0)
-          @q << data[1]
+          redis = Redis.new(:host => "localhost", :port => 6379)
+          loop do
+            if data = redis.blpop("test", 0)
+              @q << data[1]
+            end
+          end
         end
       end
     end
-  end
-end
-
-class LocalRedisWordCountTopology
-  def start(base_class_path, env)
-    builder = TopologyBuilder.new
-    builder.setSpout(1, JRubySpout.new(base_class_path, "RedisWordSpout"), 1)
-    builder.setBolt(2, JRubyBolt.new(base_class_path, "WordCountBolt"), 3).fieldsGrouping(1, Fields.new("word"))
-
-    conf = Config.new
-    conf.setDebug(true)
-    conf.setMaxTaskParallelism(3)
-
-    cluster = LocalCluster.new
-    cluster.submitTopology("redis-word-count", conf, builder.createTopology)
-    sleep(600)
-    cluster.shutdown
+
+    class LocalRedisWordCountTopology
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        builder.setSpout('RedisWordSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RedisWordSpout"), 1)
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 3).fieldsGrouping('RedisWordSpout', Fields.new("word"))
+
+        conf = Config.new
+        conf.setDebug(true)
+        conf.setMaxTaskParallelism(3)
+
+        cluster = LocalCluster.new
+        cluster.submitTopology("redis_word_count", conf, builder.createTopology)
+        sleep(600)
+        cluster.shutdown
+      end
+    end
   end
 end

data/examples/native/local_word_count_topology.rb

@@ -1,21 +1,27 @@
+require 'lib/red_storm'
 require 'examples/native/random_sentence_spout'
 require 'examples/native/split_sentence_bolt'
 require 'examples/native/word_count_bolt'
 
-class LocalWordCountTopology
-  def start(base_class_path, env)
-    builder = TopologyBuilder.new
-    builder.setSpout(1, JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
-    builder.setBolt(2, JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 8).shuffleGrouping(1)
-    builder.setBolt(3, JRubyBolt.new(base_class_path, "WordCountBolt"), 12).fieldsGrouping(2, Fields.new("word"))
 
-    conf = Config.new
-    conf.setDebug(true)
-    conf.setMaxTaskParallelism(3)
+module Examples
+  class LocalWordCountTopology
+    RedStorm::Configuration.topology_class = self
 
-    cluster = LocalCluster.new
-    cluster.submitTopology("word-count", conf, builder.createTopology)
-    sleep(5)
-    cluster.shutdown
+    def start(base_class_path, env)
+      builder = TopologyBuilder.new
+      builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
+      builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 8).shuffleGrouping('RandomSentenceSpout')
+      builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+
+      conf = Config.new
+      conf.setDebug(true)
+      conf.setMaxTaskParallelism(3)
+
+      cluster = LocalCluster.new
+      cluster.submitTopology("word_count", conf, builder.createTopology)
+      sleep(5)
+      cluster.shutdown
+    end
   end
 end

data/examples/native/random_sentence_spout.rb

@@ -1,26 +1,30 @@
-class RandomSentenceSpout
-  attr_reader :is_distributed
+module RedStorm
+  module Examples
+    class RandomSentenceSpout
+      attr_reader :is_distributed
 
-  def initialize
-    @is_distributed = true
-    @sentences = [
-      "the cow jumped over the moon",
-      "an apple a day keeps the doctor away",
-      "four score and seven years ago",
-      "snow white and the seven dwarfs",
-      "i am at two with nature"
-    ]
-  end 
+      def initialize
+        @is_distributed = true
+        @sentences = [
+          "the cow jumped over the moon",
+          "an apple a day keeps the doctor away",
+          "four score and seven years ago",
+          "snow white and the seven dwarfs",
+          "i am at two with nature"
+        ]
+      end 
 
-  def open(conf, context, collector)
-    @collector = collector
-  end
-  
-  def next_tuple
-    @collector.emit(Values.new(@sentences[rand(@sentences.length)]))
-  end
+      def open(conf, context, collector)
+        @collector = collector
+      end
+      
+      def next_tuple
+        @collector.emit(Values.new(@sentences[rand(@sentences.length)]))
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word"))
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
   end
-end
+end

data/examples/native/split_sentence_bolt.rb

@@ -1,13 +1,17 @@
-class SplitSentenceBolt
-  def prepare(conf, context, collector)
-    @collector = collector
-  end
+module RedStorm
+  module Examples
+    class SplitSentenceBolt
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
 
-  def execute(tuple)
-    tuple.getString(0).split(" ").each {|w| @collector.emit(Values.new(w)) }
-  end
+      def execute(tuple)
+        tuple.getString(0).split(" ").each {|w| @collector.emit(Values.new(w)) }
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word"))
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
   end
 end

data/examples/native/word_count_bolt.rb

@@ -1,19 +1,23 @@
-class WordCountBolt
-  def initialize
-    @counts = Hash.new{|h, k| h[k] = 0}
-  end
+module RedStorm
+  module Examples
+    class WordCountBolt
+      def initialize
+        @counts = Hash.new{|h, k| h[k] = 0}
+      end
 
-  def prepare(conf, context, collector)
-    @collector = collector
-  end
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
 
-  def execute(tuple)
-    word = tuple.getString(0)
-    @counts[word] += 1
-    @collector.emit(Values.new(word, @counts[word]))
-  end
+      def execute(tuple)
+        word = tuple.getString(0)
+        @counts[word] += 1
+        @collector.emit(Values.new(word, @counts[word]))
+      end
 
-  def declare_output_fields(declarer)
-    declarer.declare(Fields.new("word", "count"))
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word", "count"))
+      end
+    end
   end
 end

data/examples/simple/exclamation_bolt.rb

@@ -1,6 +1,10 @@
 require 'red_storm'
 
-class ExclamationBolt < RedStorm::SimpleBolt
-  output_fields :word
-  on_receive (:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
+module RedStorm
+  module Examples
+    class ExclamationBolt < RedStorm::SimpleBolt
+      output_fields :word
+      on_receive(:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
+    end
+  end
 end

data/examples/simple/exclamation_topology.rb

@@ -4,33 +4,36 @@ require 'examples/simple/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
-class ExclamationTopology < RedStorm::SimpleTopology
-  spout TestWordSpout, :parallelism => 10
-  
-  bolt ExclamationBolt, :parallelism => 3 do
-    source TestWordSpout, :shuffle
-  end
-  
-  bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
-    source ExclamationBolt, :shuffle
-  end
+module RedStorm
+  module Examples
+    class ExclamationTopology < RedStorm::SimpleTopology
+      spout TestWordSpout, :parallelism => 10
+      
+      bolt ExclamationBolt, :parallelism => 3 do
+        source TestWordSpout, :shuffle
+      end
+      
+      bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
+        source ExclamationBolt, :shuffle
+      end
 
-  configure do |env|
-    case env
-    when :local
-      debug true
-      max_task_parallelism 3
-    when :cluster
-      debug true
-      num_workers 20
-      max_spout_pending(1000);
-    end
-  end
+      configure do |env|
+        debug true
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
 
-  on_submit do |env|
-    if env == :local
-      sleep(5)
-      cluster.shutdown
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
     end
   end
 end

data/examples/simple/exclamation_topology2.rb

@@ -4,38 +4,41 @@ require 'red_storm'
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 # and a locally defined ExclamationBolt
 
-class ExclamationBolt < RedStorm::SimpleBolt
-  output_fields :word
-  on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"}
-end
+module RedStorm
+  module Examples
+    class ExclamationBolt < RedStorm::SimpleBolt
+      output_fields :word
+      on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"}
+    end
 
-class ExclamationTopology2 < RedStorm::SimpleTopology
-  spout TestWordSpout, :parallelism => 10
-  
-  bolt ExclamationBolt, :parallelism => 3 do
-    source TestWordSpout, :shuffle
-  end
-  
-  bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
-    source ExclamationBolt, :shuffle
-  end
+    class ExclamationTopology2 < RedStorm::SimpleTopology
+      spout TestWordSpout, :parallelism => 10
+      
+      bolt ExclamationBolt, :parallelism => 3 do
+        source TestWordSpout, :shuffle
+      end
+      
+      bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
+        source ExclamationBolt, :shuffle
+      end
 
-  configure do |env|
-    case env
-    when :local
-      debug true
-      max_task_parallelism 3
-    when :cluster
-      debug true
-      num_workers 20
-      max_spout_pending(1000);
-    end
-  end
+      configure do |env|
+        debug true
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
 
-  on_submit do |env|
-    if env == :local
-      sleep(5)
-      cluster.shutdown
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
     end
   end
 end

data/examples/simple/random_sentence_spout.rb

@@ -1,18 +1,22 @@
 require 'red_storm'
 
-class RandomSentenceSpout < RedStorm::SimpleSpout
-  set :is_distributed => true
-  output_fields :word
+module RedStorm
+  module Examples
+    class RandomSentenceSpout < RedStorm::SimpleSpout
+      set :is_distributed => true
+      output_fields :word
 
-  on_send {@sentences[rand(@sentences.length)]}
+      on_send {@sentences[rand(@sentences.length)]}
 
-  on_init do
-    @sentences = [
-      "the cow jumped over the moon",
-      "an apple a day keeps the doctor away",
-      "four score and seven years ago",
-      "snow white and the seven dwarfs",
-      "i am at two with nature"
-    ]
+      on_init do
+        @sentences = [
+          "the cow jumped over the moon",
+          "an apple a day keeps the doctor away",
+          "four score and seven years ago",
+          "snow white and the seven dwarfs",
+          "i am at two with nature"
+        ]
+      end
+    end
   end
-end
+end

data/examples/simple/redis_word_count_topology.rb

@@ -5,51 +5,55 @@ require 'red_storm'
 
 require 'examples/simple/word_count_bolt'
 
-# RedisWordSpout reads the Redis queue "test" on localhost:6379 
-# and emits each word items pop'ed from the queue.
+module RedStorm
+  module Examples
 
-class RedisWordSpout < RedStorm::SimpleSpout
-  output_fields :word
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # and emits each word items pop'ed from the queue.
 
-  on_send {@q.pop if @q.size > 0}
+    class RedisWordSpout < RedStorm::SimpleSpout
+      output_fields :word
 
-  on_init do
-    @q = Queue.new
-    @redis_reader = detach_redis_reader
-  end
-  
-  private
-
-  def detach_redis_reader
-    Thread.new do
-      Thread.current.abort_on_exception = true
+      on_send {@q.pop if @q.size > 0}
 
-      redis = Redis.new(:host => "localhost", :port => 6379)
-      loop do
-        if data = redis.blpop("test", 0)
-          @q << data[1]
+      on_init do
+        @q = Queue.new
+        @redis_reader = detach_redis_reader
+      end
+      
+      private
+
+      def detach_redis_reader
+        Thread.new do
+          Thread.current.abort_on_exception = true
+
+          redis = Redis.new(:host => "localhost", :port => 6379)
+          loop do
+            if data = redis.blpop("test", 0)
+              @q << data[1]
+            end
+          end
         end
       end
     end
-  end
-end
 
-class RedisWordCountTopology < RedStorm::SimpleTopology
-  spout RedisWordSpout
-      
-  bolt WordCountBolt, :parallelism => 3 do
-    source RedisWordSpout, :fields => ["word"]
-  end
+    class RedisWordCountTopology < RedStorm::SimpleTopology
+      spout RedisWordSpout
+          
+      bolt WordCountBolt, :parallelism => 3 do
+        source RedisWordSpout, :fields => ["word"]
+      end
 
-  configure do |env|
-    case env
-    when :local
-      debug true
-      max_task_parallelism 3
-    when :cluster
-      debug true
-      num_workers 20
-      max_spout_pending(1000);
+      configure do |env|
+        debug true
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
     end
   end
 end

data/examples/simple/split_sentence_bolt.rb

@@ -1,29 +1,33 @@
 require 'red_storm'
 
-class SplitSentenceBolt < RedStorm::SimpleBolt
-  output_fields :word
+module RedStorm
+  module Examples
+    class SplitSentenceBolt < RedStorm::SimpleBolt
+      output_fields :word
 
-  # block declaration style using auto-emit (default)
-  #
-  on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}}
+      # block declaration style using auto-emit (default)
+      #
+      on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}}
 
-  # block declaration style no auto-emit
-  #
-  # on_receive :emit => false do |tuple| 
-  #   tuple.getString(0).split(' ').each{|w| unanchored_emit(w)}
-  # end
+      # block declaration style no auto-emit
+      #
+      # on_receive :emit => false do |tuple| 
+      #   tuple.getString(0).split(' ').each{|w| unanchored_emit(w)}
+      # end
 
-  # alternate declaration style using on_receive method
-  #
-  # on_receive :emit => true
-  # def on_receive(tuple)
-  #   tuple.getString(0).split(' ').map{|w| [w]}
-  # end
+      # alternate declaration style using on_receive method
+      #
+      # on_receive :emit => true
+      # def on_receive(tuple)
+      #   tuple.getString(0).split(' ').map{|w| [w]}
+      # end
 
-  # alternate declaration style using any specific method
-  #
-  # on_receive :my_method, :emit => true
-  # def my_method(tuple)
-  #   tuple.getString(0).split(' ').map{|w| [w]}
-  # end
+      # alternate declaration style using any specific method
+      #
+      # on_receive :my_method, :emit => true
+      # def my_method(tuple)
+      #   tuple.getString(0).split(' ').map{|w| [w]}
+      # end
+    end
+  end
 end

data/examples/simple/word_count_bolt.rb

@@ -1,15 +1,19 @@
 require 'red_storm'
 
-class WordCountBolt < RedStorm::SimpleBolt
-  output_fields :word, :count
-  on_init {@counts = Hash.new{|h, k| h[k] = 0}}
+module RedStorm
+  module Examples
+    class WordCountBolt < RedStorm::SimpleBolt
+      output_fields :word, :count
+      on_init {@counts = Hash.new{|h, k| h[k] = 0}}
 
-  # block declaration style using auto-emit (default)
-  #
-  on_receive do |tuple|
-    word = tuple.getString(0)
-    @counts[word] += 1
+      # block declaration style using auto-emit (default)
+      #
+      on_receive do |tuple|
+        word = tuple.getString(0)
+        @counts[word] += 1
 
-    [word, @counts[word]]
+        [word, @counts[word]]
+      end
+    end
   end
 end

data/examples/simple/word_count_topology.rb

@@ -2,33 +2,36 @@ require 'examples/simple/random_sentence_spout'
 require 'examples/simple/split_sentence_bolt'
 require 'examples/simple/word_count_bolt'
 
-class WordCountTopology < RedStorm::SimpleTopology
-  spout RandomSentenceSpout, :parallelism => 5
-  
-  bolt SplitSentenceBolt, :parallelism => 8 do
-    source RandomSentenceSpout, :shuffle
-  end
-  
-  bolt WordCountBolt, :parallelism => 12 do
-    source SplitSentenceBolt, :fields => ["word"]
-  end
+module RedStorm
+  module Examples
+    class WordCountTopology < SimpleTopology
+      spout RandomSentenceSpout, :parallelism => 5
+      
+      bolt SplitSentenceBolt, :parallelism => 8 do
+        source RandomSentenceSpout, :shuffle
+      end
+      
+      bolt WordCountBolt, :parallelism => 12 do
+        source SplitSentenceBolt, :fields => ["word"]
+      end
 
-  configure :word_count do |env|
-    case env
-    when :local
-      debug true
-      max_task_parallelism 3
-    when :cluster
-      debug true
-      num_workers 20
-      max_spout_pending(1000);
-    end
-  end
+      configure :word_count do |env|
+        debug true
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
 
-  on_submit do |env|
-    if env == :local
-      sleep(5)
-      cluster.shutdown
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
     end
   end
 end

data/lib/red_storm.rb

@@ -3,6 +3,7 @@ module RedStorm
 end
 
 require 'red_storm/version'
+require 'red_storm/configuration'
 require 'red_storm/application'
 require 'red_storm/simple_bolt'
 require 'red_storm/simple_spout'

data/lib/red_storm/configuration.rb

@@ -0,0 +1,16 @@
+module RedStorm
+  module Configuration
+    extend self
+
+    @topology_class = nil
+
+    def topology_class=(clazz)
+      @topology_class = clazz
+    end
+
+    def topology_class
+      @topology_class
+    end
+
+  end
+end

data/lib/red_storm/simple_topology.rb

@@ -1,3 +1,5 @@
+require 'red_storm/configuration'
+
 module RedStorm
 
   class SimpleTopology
@@ -8,11 +10,11 @@ module RedStorm
 
     class ComponentDefinition
       attr_reader :clazz, :parallelism
-      attr_accessor :id
+      attr_accessor :id # ids are forced to string
 
       def initialize(component_class, id, parallelism)
         @clazz = component_class
-        @id = id
+        @id = id.to_s
         @parallelism = parallelism
       end
     end
@@ -28,16 +30,16 @@ module RedStorm
       end
 
       def source(source_id, grouping)
-        @sources << [source_id.is_a?(Class) ? SimpleTopology.underscore(source_id) : source_id, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
+        @sources << [source_id.is_a?(Class) ? SimpleTopology.underscore(source_id) : source_id.to_s, grouping.is_a?(Hash) ? grouping : {grouping => nil}]
       end
 
       def define_grouping(declarer)
         @sources.each do |source_id, grouping|
           grouper, params = grouping.first
-
+            # declarer.fieldsGrouping(source_id, Fields.new())
           case grouper
           when :fields
-            declarer.fieldsGrouping(source_id, Fields.new(*params))
+            declarer.fieldsGrouping(source_id, Fields.new(*([params].flatten.map(&:to_s))))
           when :global
             declarer.globalGrouping(source_id)
           when :shuffle
@@ -88,6 +90,7 @@ module RedStorm
     end
 
     def self.configure(name = nil, &configure_block)
+      Configuration.topology_class = self
       @topology_name = name if name
       @configure_block = configure_block if block_given?
     end
@@ -131,31 +134,13 @@ module RedStorm
     private
 
     def self.resolve_ids!(components)
-      next_numeric_id = 1
-      resolved_names = {}
-
-      numeric_components, symbolic_components = components.partition{|c| c.id.is_a?(Fixnum)}
-      numeric_ids = numeric_components.map(&:id)
-
-      # assign numeric ids to symbolic ids
-      symbolic_components.each do |component|
-        id = component.id.to_s
-        raise("duplicate symbolic id in #{component.clazz.name} on id=#{id}") if resolved_names.has_key?(id)
-        next_numeric_id += 1 while numeric_ids.include?(next_numeric_id)
-        numeric_ids << next_numeric_id
-        resolved_names[id] = next_numeric_id
-      end
-
-      # reassign numeric ids to all components
-      components.each do |component|
-        unless component.id.is_a?(Fixnum)
-          component.id = resolved_names[component.id.to_s] || raise("cannot resolve #{component.clazz.name} id=#{component.id.to_s}")
-        end
+      # verify duplicate implicit ids
+      ids = components.map(&:id)
+      components.reverse.each do |component|
+        raise("duplicate id in #{component.clazz.name} on id=#{component.id}") if ids.select{|id| id == component.id}.size > 1
+        # verify source_id references
         if component.respond_to?(:sources)
-          component.sources.map! do |source_id, grouping|
-            id = source_id.is_a?(Fixnum) ? source_id : resolved_names[source_id.to_s] || raise("cannot resolve #{component.clazz.name} source id=#{source_id.to_s}")
-            [id, grouping]
-          end
+          component.sources.each{|source_id, grouping| raise("cannot resolve #{component.clazz.name} source id=#{source_id}") unless ids.include?(source_id)}
         end
       end
     end

data/lib/red_storm/topology_launcher.rb

@@ -24,7 +24,7 @@ java_import 'redstorm.storm.jruby.JRubySpout'
 java_package 'redstorm'
 
 # TopologyLauncher is the application entry point when launching a topology. Basically it will 
-# call require on the specified Ruby topology/project class file path and call its start method
+# call require on the specified Ruby topology class file path and call its start method
 class TopologyLauncher
 
   java_signature 'void main(String[])'
@@ -35,12 +35,12 @@ class TopologyLauncher
     end
     env = args[0].to_sym
     class_path = args[1]
-    clazz = camel_case(class_path.split('/').last.split('.').first)
-
-    puts("RedStorm v#{RedStorm::VERSION} starting topology #{clazz} in #{env.to_s} environment")
 
     require class_path
-    Object.module_eval(clazz).new.start(class_path, env)
+
+    topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
+    puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
+    RedStorm::Configuration.topology_class.new.start(class_path, env)
   end
 
   private 

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.2.1'
+  VERSION = '0.4.0'
 end

data/lib/tasks/red_storm.rake

@@ -15,6 +15,7 @@ TARGET_SRC_DIR = "#{TARGET_DIR}/src"
 TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"  
 TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency"
 TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked"
+TARGET_MARKERS_DIR = "#{TARGET_DIR}/dependency-markers"
 TARGET_GEMS_DIR = "#{TARGET_DIR}/gems"
 TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar"
 
@@ -55,7 +56,7 @@ task :install => [:deps, :build] do
 end
 
 task :unpack do
-  system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR}")
+  system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
 end
 
 task :devjar => [:unpack, :clean_jar] do
@@ -111,7 +112,7 @@ task :examples do
 end
 
 task :deps do
-  system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR}")
+  system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
 end
 
 task :build => :setup do
@@ -148,5 +149,5 @@ end
 
 def build_jruby(source_path)
   puts("\n--> Compiling JRuby")
-  system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.5.3.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
+  system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.6.2.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
 end

data/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>redstorm</groupId>
   <artifactId>redstorm</artifactId>
-  <version>0.2.1</version>
+  <version>0.4.0</version>
   <name>RedStorm JRuby on Storm</name>
 
   <properties>
@@ -27,13 +27,13 @@
     <dependency>
       <groupId>storm</groupId>
       <artifactId>storm</artifactId>
-      <version>0.5.4</version>
+      <version>0.6.2</version>
     </dependency>
   
     <dependency>
       <groupId>org.jruby</groupId>
       <artifactId>jruby-complete</artifactId>
-      <version>1.6.5</version>
+      <version>1.6.6</version>
     </dependency>
   </dependencies>
 
@@ -48,11 +48,12 @@
             <artifactItem>
               <groupId>org.jruby</groupId>
               <artifactId>jruby-complete</artifactId>
-              <version>1.6.5</version>
+              <version>1.6.6</version>
               <type>jar</type>
               <overWrite>false</overWrite>
             </artifactItem>
           </artifactItems>
+          <markersDirectory>${markersDirectory}</markersDirectory>
         </configuration>
         <executions>
           <execution>
@@ -66,4 +67,4 @@
     </plugins>
   </build>
 
-</project>
+</project>

metadata

@@ -2,7 +2,7 @@
 name: redstorm
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.1
+  version: 0.4.0
 platform: ruby
 authors: 
   - Colin Surprenant
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-23 00:00:00 Z
+date: 2012-02-08 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: rubyforge
@@ -57,6 +57,7 @@ extra_rdoc_files: []
 files: 
   - lib/red_storm.rb
   - lib/red_storm/application.rb
+  - lib/red_storm/configuration.rb
   - lib/red_storm/simple_bolt.rb
   - lib/red_storm/simple_spout.rb
   - lib/red_storm/simple_topology.rb
@@ -114,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: redstorm
-rubygems_version: 1.8.9
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: JRuby on Storm