redstorm 0.1.1 → 0.2.0

data/Rakefile

@@ -1,129 +1,14 @@
-require 'ant'
+load 'lib/tasks/red_storm.rake'
 
-begin
-  # will work from gem, since lib dir is in gem require_paths
-  require 'red_storm'
-rescue LoadError
-  # will work within RedStorm dev project
-  $:.unshift './lib'
-  require 'red_storm'
-end
-
-CWD = Dir.pwd
-TARGET_DIR = "#{CWD}/target"
-TARGET_SRC_DIR = "#{TARGET_DIR}/src"
-TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"  
-TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency"
-TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked"
-TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar"
-
-JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main"
-JRUBY_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/lib/red_storm"
-
-SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples"
-DST_EXAMPLES = "#{CWD}/examples"
-  
-task :default => [:clean, :build]
-
-task :launch, :class_file do |t, args|
-  gem_home = ENV["GEM_HOME"].to_s.empty? ? " -Djruby.gem.home=`gem env home`" : ""
-  puts("launching java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:class_file]}")
-  system("java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:class_file]}")
-end
-
-task :clean do
-  ant.delete :dir => TARGET_DIR
-end
-
-task :clean_jar do
-  ant.delete :dir => "#{TARGET_DIR}/cluster-topology.jar"
-end
-
-task :setup do  
-  ant.mkdir :dir => TARGET_DIR 
-  ant.mkdir :dir => TARGET_CLASSES_DIR 
-  ant.mkdir :dir => TARGET_SRC_DIR
-  ant.path :id => 'classpath' do  
-    fileset :dir => TARGET_DEPENDENCY_DIR  
-    fileset :dir => TARGET_CLASSES_DIR  
-  end  
-end
-
-task :install => [:deps, :build] do
-  puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}")
-end
-
-task :unpack do
-  system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR}")
-end
+task :default => :spec
 
-task :jar => [:unpack, :clean_jar] do
-  ant.jar :destfile => TARGET_CLUSTER_JAR do
-    fileset :dir => TARGET_CLASSES_DIR
-    fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
-    fileset :dir => CWD do
-      exclude :name => "target/**/*"
-    end
-    manifest do
-      attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
-    end
-  end
-  puts("\nRedStorm jar completed. Generated jar file #{TARGET_CLUSTER_JAR}")
-end
-
-task :examples do
-  if File.identical?(SRC_EXAMPLES, DST_EXAMPLES)
-    STDERR.puts("error: cannot copy examples into itself")
-    exit(1)
-  end
-  if File.exist?(DST_EXAMPLES)
-    STDERR.puts("error: directory #{DST_EXAMPLES} already exists")
-    exit(1)
+begin
+  require 'rspec/core/rake_task'
+  desc "run specs"
+  task :spec do
+    system("ruby -v")
+    RSpec::Core::RakeTask.new
   end
-
-  puts("copying examples into #{DST_EXAMPLES}")
-  system("mkdir #{DST_EXAMPLES}")
-  system("cp -r #{SRC_EXAMPLES}/* #{DST_EXAMPLES}")
-  puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}")
-end
-
-task :deps do
-  system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR}")
-end
-
-task :build => :setup do
-  # compile the JRuby proxy classes to Java
-  build_jruby("#{JRUBY_SRC_DIR}/proxy")
-
-  # compile the generated Java proxy classes
-  build_java_dir("#{TARGET_SRC_DIR}")
-
-  # generate the JRuby topology launcher
-  build_jruby("#{JRUBY_SRC_DIR}/topology_launcher.rb")
-
-  # compile the JRuby proxy classes
-  build_java_dir("#{JAVA_SRC_DIR}")
-
-  # compile the JRuby proxy classes
-  build_java_dir("#{TARGET_SRC_DIR}")
-end 
-
-def build_java_dir(source_folder)
-  puts("\n--> Compiling Java")
-  ant.javac(
-    :srcdir => source_folder,
-    :destdir => TARGET_CLASSES_DIR,
-    :classpathref => 'classpath', 
-    :source => "1.6",
-    :target => "1.6",
-    :debug => "yes",
-    :includeantruntime => "no",
-    :verbose => false,
-    :listfiles => true
-  )
-end  
-
-def build_jruby(source_path)
-  puts("\n--> Compiling JRuby")
-  system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.5.3.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
+rescue NameError, LoadError => e
+  puts e
 end

data/bin/redstorm

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'rubygems' 
+require 'rake'
 
 begin
   # will work from gem, since lib dir is in gem require_paths

data/examples/{cluster_word_count_topology.rb → native/cluster_word_count_topology.rb}

@@ -1,9 +1,9 @@
-require 'examples/random_sentence_spout'
-require 'examples/split_sentence_bolt'
-require 'examples/word_count_bolt'
+require 'examples/native/random_sentence_spout'
+require 'examples/native/split_sentence_bolt'
+require 'examples/native/word_count_bolt'
 
 class ClusterWordCountTopology
-  def start(base_class_path)
+  def start(base_class_path, env)
     builder = TopologyBuilder.new
     builder.setSpout(1, JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
     builder.setBolt(2, JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 4).shuffleGrouping(1)

data/examples/{local_exclamation_topology.rb → native/local_exclamation_topology.rb}

@@ -1,10 +1,10 @@
 java_import 'backtype.storm.testing.TestWordSpout'
-require 'examples/exclamation_bolt'
+require 'examples/native/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
 class LocalExclamationTopology
-  def start(base_class_path)
+  def start(base_class_path, env)
     builder = TopologyBuilder.new
     
     builder.setSpout(1, TestWordSpout.new, 10)     

data/examples/{local_exclamation_topology2.rb → native/local_exclamation_topology2.rb}

@@ -18,7 +18,7 @@ end
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
 class LocalExclamationTopology2
-  def start(base_class_path)
+  def start(base_class_path, env)
     builder = TopologyBuilder.new
     
     builder.setSpout(1, TestWordSpout.new, 10)     

data/examples/{local_redis_word_count_topology.rb → native/local_redis_word_count_topology.rb}

@@ -1,6 +1,6 @@
 require 'redis'
 require 'thread'
-require 'examples/word_count_bolt'
+require 'examples/native/word_count_bolt'
 
 # RedisWordSpout reads the Redis queue "test" on localhost:6379 
 # and emits each word items pop'ed from the queue.
@@ -41,7 +41,7 @@ class RedisWordSpout
 end
 
 class LocalRedisWordCountTopology
-  def start(base_class_path)
+  def start(base_class_path, env)
     builder = TopologyBuilder.new
     builder.setSpout(1, JRubySpout.new(base_class_path, "RedisWordSpout"), 1)
     builder.setBolt(2, JRubyBolt.new(base_class_path, "WordCountBolt"), 3).fieldsGrouping(1, Fields.new("word"))

data/examples/{local_word_count_topology.rb → native/local_word_count_topology.rb}

@@ -1,9 +1,9 @@
-require 'examples/random_sentence_spout'
-require 'examples/split_sentence_bolt'
-require 'examples/word_count_bolt'
+require 'examples/native/random_sentence_spout'
+require 'examples/native/split_sentence_bolt'
+require 'examples/native/word_count_bolt'
 
 class LocalWordCountTopology
-  def start(base_class_path)
+  def start(base_class_path, env)
     builder = TopologyBuilder.new
     builder.setSpout(1, JRubySpout.new(base_class_path, "RandomSentenceSpout"), 5)
     builder.setBolt(2, JRubyBolt.new(base_class_path, "SplitSentenceBolt"), 8).shuffleGrouping(1)

data/examples/simple/exclamation_bolt.rb

@@ -0,0 +1,6 @@
+require 'red_storm'
+
+class ExclamationBolt < RedStorm::SimpleBolt
+  output_fields :word
+  on_receive (:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
+end

data/examples/simple/exclamation_topology.rb

@@ -0,0 +1,36 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'examples/simple/exclamation_bolt'
+
+# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+
+class ExclamationTopology < RedStorm::SimpleTopology
+  spout TestWordSpout, :parallelism => 10
+  
+  bolt ExclamationBolt, :parallelism => 3 do
+    source TestWordSpout, :shuffle
+  end
+  
+  bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
+    source ExclamationBolt, :shuffle
+  end
+
+  configure do |env|
+    case env
+    when :local
+      debug true
+      max_task_parallelism 3
+    when :cluster
+      debug true
+      num_workers 20
+      max_spout_pending(1000);
+    end
+  end
+
+  on_submit do |env|
+    if env == :local
+      sleep(5)
+      cluster.shutdown
+    end
+  end
+end

data/examples/simple/exclamation_topology2.rb

@@ -0,0 +1,41 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+require 'red_storm'
+
+# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+# and a locally defined ExclamationBolt
+
+class ExclamationBolt < RedStorm::SimpleBolt
+  output_fields :word
+  on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"}
+end
+
+class ExclamationTopology2 < RedStorm::SimpleTopology
+  spout TestWordSpout, :parallelism => 10
+  
+  bolt ExclamationBolt, :parallelism => 3 do
+    source TestWordSpout, :shuffle
+  end
+  
+  bolt ExclamationBolt, :id => :ignore, :parallelism => 2 do
+    source ExclamationBolt, :shuffle
+  end
+
+  configure do |env|
+    case env
+    when :local
+      debug true
+      max_task_parallelism 3
+    when :cluster
+      debug true
+      num_workers 20
+      max_spout_pending(1000);
+    end
+  end
+
+  on_submit do |env|
+    if env == :local
+      sleep(5)
+      cluster.shutdown
+    end
+  end
+end

data/examples/simple/random_sentence_spout.rb

@@ -0,0 +1,18 @@
+require 'red_storm'
+
+class RandomSentenceSpout < RedStorm::SimpleSpout
+  set :is_distributed => true
+  output_fields :word
+
+  on_send {@sentences[rand(@sentences.length)]}
+
+  on_init do
+    @sentences = [
+      "the cow jumped over the moon",
+      "an apple a day keeps the doctor away",
+      "four score and seven years ago",
+      "snow white and the seven dwarfs",
+      "i am at two with nature"
+    ]
+  end
+end

data/examples/simple/redis_word_count_topology.rb

@@ -0,0 +1,54 @@
+require 'redis'
+require 'thread'
+require 'red_storm'
+
+require 'examples/simple/word_count_bolt'
+
+# RedisWordSpout reads the Redis queue "test" on localhost:6379 
+# and emits each word items pop'ed from the queue.
+
+class RedisWordSpout < RedStorm::SimpleSpout
+  output_fields :word
+
+  on_send {@q.pop if @q.size > 0}
+
+  on_init do
+    @q = Queue.new
+    @redis_reader = detach_redis_reader
+  end
+  
+  private
+
+  def detach_redis_reader
+    Thread.new do
+      Thread.current.abort_on_exception = true
+
+      redis = Redis.new(:host => "localhost", :port => 6379)
+      loop do
+        if data = redis.blpop("test", 0)
+          @q << data[1]
+        end
+      end
+    end
+  end
+end
+
+class RedisWordCountTopology < RedStorm::SimpleTopology
+  spout RedisWordSpout
+      
+  bolt WordCountBolt, :parallelism => 3 do
+    source RedisWordSpout, :fields => ["word"]
+  end
+
+  configure do |env|
+    case env
+    when :local
+      debug true
+      max_task_parallelism 3
+    when :cluster
+      debug true
+      num_workers 20
+      max_spout_pending(1000);
+    end
+  end
+end

data/examples/simple/split_sentence_bolt.rb

@@ -0,0 +1,29 @@
+require 'red_storm'
+
+class SplitSentenceBolt < RedStorm::SimpleBolt
+  output_fields :word
+
+  # block declaration style using auto-emit (default)
+  #
+  on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}}
+
+  # block declaration style no auto-emit
+  #
+  # on_receive :emit => false do |tuple| 
+  #   tuple.getString(0).split(' ').each{|w| unanchored_emit(w)}
+  # end
+
+  # alternate declaration style using on_receive method
+  #
+  # on_receive :emit => true
+  # def on_receive(tuple)
+  #   tuple.getString(0).split(' ').map{|w| [w]}
+  # end
+
+  # alternate declaration style using any specific method
+  #
+  # on_receive :my_method, :emit => true
+  # def my_method(tuple)
+  #   tuple.getString(0).split(' ').map{|w| [w]}
+  # end
+end

data/examples/simple/word_count_bolt.rb

@@ -0,0 +1,15 @@
+require 'red_storm'
+
+class WordCountBolt < RedStorm::SimpleBolt
+  output_fields :word, :count
+  on_init {@counts = Hash.new{|h, k| h[k] = 0}}
+
+  # block declaration style using auto-emit (default)
+  #
+  on_receive do |tuple|
+    word = tuple.getString(0)
+    @counts[word] += 1
+
+    [word, @counts[word]]
+  end
+end

data/examples/simple/word_count_topology.rb

@@ -0,0 +1,34 @@
+require 'examples/simple/random_sentence_spout'
+require 'examples/simple/split_sentence_bolt'
+require 'examples/simple/word_count_bolt'
+
+class WordCountTopology < RedStorm::SimpleTopology
+  spout RandomSentenceSpout, :parallelism => 5
+  
+  bolt SplitSentenceBolt, :parallelism => 8 do
+    source RandomSentenceSpout, :shuffle
+  end
+  
+  bolt WordCountBolt, :parallelism => 12 do
+    source SplitSentenceBolt, :fields => ["word"]
+  end
+
+  configure :word_count do |env|
+    case env
+    when :local
+      debug true
+      max_task_parallelism 3
+    when :cluster
+      debug true
+      num_workers 20
+      max_spout_pending(1000);
+    end
+  end
+
+  on_submit do |env|
+    if env == :local
+      sleep(5)
+      cluster.shutdown
+    end
+  end
+end