kb-redstorm 0.6.4

data/examples/shell/shell_topology.rb

@@ -0,0 +1,41 @@
+require 'red_storm'
+require 'thread'
+
+java_import 'redstorm.storm.jruby.JRubyShellBolt'
+
+class SimpleSpout < RedStorm::SimpleSpout
+  on_init do
+    @q = Queue.new
+    @q << "the quick red fox"
+  end
+
+  on_send do
+    # avoid putting the thread to sleep endlessly on @q.pop which will prevent local cluster.shutdown
+    sleep(1)
+    @q.pop unless @q.empty?
+  end
+end
+
+class ShellTopology < RedStorm::SimpleTopology
+  spout SimpleSpout do
+    output_fields :string
+  end
+
+  bolt JRubyShellBolt, ["python", "splitsentence.py"] do
+    output_fields "word"
+    source SimpleSpout, :shuffle
+  end
+
+  configure do |env|
+    debug true
+  end
+
+  on_submit do |env|
+    case env
+    when :local
+      sleep(10)
+      cluster.shutdown
+    end
+  end
+end
+

data/examples/simple/exclamation_bolt.rb

@@ -0,0 +1,10 @@
+require 'red_storm'
+
+module RedStorm
+  module Examples
+    class ExclamationBolt < RedStorm::SimpleBolt
+      output_fields :word
+      on_receive(:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
+    end
+  end
+end

data/examples/simple/exclamation_topology.rb

@@ -0,0 +1,45 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'examples/simple/exclamation_bolt'
+
+# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+
+module RedStorm
+  module Examples
+    class ExclamationTopology < RedStorm::SimpleTopology
+      spout TestWordSpout, :parallelism => 5 do
+        debug true
+      end
+      
+      bolt ExclamationBolt, :parallelism => 2 do
+        source TestWordSpout, :shuffle
+        # max_task_parallelism 1
+      end
+      
+      bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
+        source ExclamationBolt, :shuffle
+        # max_task_parallelism 1
+        debug true
+      end
+
+      configure do |env|
+        debug false
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        case env
+        when :local
+          max_task_parallelism 40
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
+
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
+    end
+  end
+end

data/examples/simple/exclamation_topology2.rb

@@ -0,0 +1,45 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+require 'red_storm'
+
+# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+# and a locally defined ExclamationBolt
+
+module RedStorm
+  module Examples
+    class ExclamationBolt < RedStorm::SimpleBolt
+      output_fields :word
+      on_receive(:ack => true, :anchor => true) {|tuple| "!#{tuple.getString(0)}!"}
+    end
+
+    class ExclamationTopology2 < RedStorm::SimpleTopology
+      spout TestWordSpout, :parallelism => 10
+      
+      bolt ExclamationBolt, :parallelism => 3 do
+        source TestWordSpout, :shuffle
+      end
+      
+      bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
+        source ExclamationBolt, :shuffle
+      end
+
+      configure do |env|
+        debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
+
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
+    end
+  end
+end

data/examples/simple/kafka_topology.rb

@@ -0,0 +1,55 @@
+require 'red_storm'
+java_import 'storm.kafka.KafkaConfig'
+java_import 'storm.kafka.SpoutConfig'
+java_import 'storm.kafka.StringScheme'
+java_import 'storm.kafka.KafkaSpout'
+
+# the KafkaTopology obviously requires a Kafka server running, you can ajust the
+# host and port below. 
+#
+# custom dependencies are required for the Kafka and Scala jars. put the following
+# dependencies in the "Dependencies" file in the root of your RedStorm project:
+#
+# {
+#   :storm_artifacts => [
+#     "storm:storm:0.8.1, transitive=true",
+#   ],
+#   :topology_artifacts => [
+#     "org.jruby:jruby-complete:1.6.8, transitive=false",
+#     "org.scala-lang:scala-library:2.8.0, transitive=false",
+#     "storm:kafka:0.7.0-incubating, transitive=false",
+#     "storm:storm-kafka:0.8.0-wip4, transitive=false",
+#   ],
+# }
+
+class KafkaTopology < RedStorm::SimpleTopology
+  spout_config = SpoutConfig.new(
+    KafkaConfig::ZkHosts.new("localhost:2181", "/brokers"),
+    "words",        # topic to read from
+    "/kafkastorm",  # Zookeeper root path to store the consumer offsets
+    "discovery"     # Zookeeper consumer id to store the consumer offsets
+  )
+  spout_config.scheme = StringScheme.new
+
+  class SplitStringBolt < RedStorm::SimpleBolt
+    on_receive {|tuple| tuple.getString(0).split.map{|w| [w]}}
+  end
+
+  spout KafkaSpout, [spout_config]
+      
+  bolt SplitStringBolt do
+    output_fields :word
+    source KafkaSpout, :shuffle
+  end
+
+  configure do |env|
+    debug true
+  end
+
+  on_submit do |env|
+    if env == :local
+      sleep(10)
+      cluster.shutdown
+    end
+  end
+end

data/examples/simple/random_sentence_spout.rb

@@ -0,0 +1,21 @@
+require 'red_storm'
+
+module RedStorm
+  module Examples
+    class RandomSentenceSpout < RedStorm::SimpleSpout
+      output_fields :word
+
+      on_send {@sentences[rand(@sentences.length)]}
+
+      on_init do
+        @sentences = [
+          "the cow jumped over the moon",
+          "an apple a day keeps the doctor away",
+          "four score and seven years ago",
+          "snow white and the seven dwarfs",
+          "i am at two with nature"
+        ]
+      end
+    end
+  end
+end

data/examples/simple/redis_word_count_topology.rb

@@ -0,0 +1,61 @@
+require 'rubygems'
+require 'red_storm'
+
+require 'redis'
+require 'thread'
+require 'examples/simple/word_count_bolt'
+
+module RedStorm
+  module Examples
+
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # and emits each word items pop'ed from the queue.
+
+    class RedisWordSpout < RedStorm::SimpleSpout
+      output_fields :word
+
+      on_send {@q.pop.to_s if @q.size > 0}
+
+      on_init do
+        @q = Queue.new
+        @redis_reader = detach_redis_reader
+      end
+      
+      private
+
+      def detach_redis_reader
+        Thread.new do
+          Thread.current.abort_on_exception = true
+
+          redis = Redis.new(:host => "localhost", :port => 6379)
+          loop do
+            if data = redis.blpop("test", 0)
+              @q << data[1]
+            end
+          end
+        end
+      end
+    end
+
+    class RedisWordCountTopology < RedStorm::SimpleTopology
+      spout RedisWordSpout
+          
+      bolt WordCountBolt, :parallelism => 3 do
+        source RedisWordSpout, :fields => ["word"]
+      end
+
+      configure do |env|
+        debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          max_task_parallelism 5
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
+    end
+  end
+end

data/examples/simple/ruby_version_topology.rb

@@ -0,0 +1,32 @@
+require 'red_storm'
+
+# this example topology only prints the Ruby version string. No tuple is emitted.
+
+module RedStorm
+  module Examples
+    class VersionSpout < RedStorm::SimpleSpout
+      output_fields :dummy
+      on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")}
+      on_send {}
+    end
+
+    class RubyVersionTopology < RedStorm::SimpleTopology
+      spout VersionSpout
+            
+      configure do |env|
+        debug true
+
+        # force the JRuby version property for this topology. this will only affect remote cluster execution
+        # for local execution use the --1.8|--1.9 switch when launching
+        # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+      end
+
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
+    end
+  end
+end

data/examples/simple/split_sentence_bolt.rb

@@ -0,0 +1,33 @@
+require 'red_storm'
+
+module RedStorm
+  module Examples
+    class SplitSentenceBolt < RedStorm::SimpleBolt
+      output_fields :word
+
+      # block declaration style using auto-emit (default)
+      #
+      on_receive {|tuple| tuple.getString(0).split(' ').map{|w| [w]}}
+
+      # block declaration style no auto-emit
+      #
+      # on_receive :emit => false do |tuple| 
+      #   tuple.getString(0).split(' ').each{|w| unanchored_emit(w)}
+      # end
+
+      # alternate declaration style using on_receive method
+      #
+      # on_receive :emit => true
+      # def on_receive(tuple)
+      #   tuple.getString(0).split(' ').map{|w| [w]}
+      # end
+
+      # alternate declaration style using any specific method
+      #
+      # on_receive :my_method, :emit => true
+      # def my_method(tuple)
+      #   tuple.getString(0).split(' ').map{|w| [w]}
+      # end
+    end
+  end
+end

data/examples/simple/word_count_bolt.rb

@@ -0,0 +1,19 @@
+require 'red_storm'
+
+module RedStorm
+  module Examples
+    class WordCountBolt < RedStorm::SimpleBolt
+      output_fields :word, :count
+      on_init {@counts = Hash.new{|h, k| h[k] = 0}}
+
+      # block declaration style using auto-emit (default)
+      #
+      on_receive do |tuple|
+        word = tuple.getValue(0).to_s
+        @counts[word] += 1
+
+        [word, @counts[word]]
+      end
+    end
+  end
+end

data/examples/simple/word_count_topology.rb

@@ -0,0 +1,38 @@
+require 'examples/simple/random_sentence_spout'
+require 'examples/simple/split_sentence_bolt'
+require 'examples/simple/word_count_bolt'
+
+module RedStorm
+  module Examples
+    class WordCountTopology < SimpleTopology
+      spout RandomSentenceSpout, :parallelism => 5
+      
+      bolt SplitSentenceBolt, :parallelism => 8 do
+        source RandomSentenceSpout, :shuffle
+      end
+      
+      bolt WordCountBolt, :parallelism => 12 do
+        source SplitSentenceBolt, :fields => ["word"]
+      end
+
+      configure :word_count do |env|
+        debug true
+        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        case env
+        when :local
+          max_task_parallelism 3
+        when :cluster
+          num_workers 20
+          max_spout_pending(1000);
+        end
+      end
+
+      on_submit do |env|
+        if env == :local
+          sleep(5)
+          cluster.shutdown
+        end
+      end
+    end
+  end
+end

data/ivy/settings.xml

@@ -0,0 +1,11 @@
+<ivysettings>
+  <settings defaultResolver="repositories"/>
+  <resolvers>
+    <chain name="repositories">
+      <ibiblio name="ibiblio" m2compatible="true"/>
+      <ibiblio name="maven2" root="http://repo.maven.apache.org/maven2/" m2compatible="true"/> 
+      <ibiblio name="sonatype" root="http://repo.maven.apache.org/maven2/" m2compatible="true"/> 
+      <ibiblio name="clojars" root="http://clojars.org/repo/" m2compatible="true"/> 
+    </chain>
+  </resolvers>
+</ivysettings>

data/lib/red_storm.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+
+require 'red_storm/environment'
+require 'red_storm/version'
+require 'red_storm/configuration'
+require 'red_storm/simple_bolt'
+require 'red_storm/simple_spout'
+require 'red_storm/simple_topology'
+require 'red_storm/simple_drpc_topology'

data/lib/red_storm/application.rb

@@ -0,0 +1,85 @@
+require 'red_storm/version'
+require 'red_storm/environment'
+
+CWD = Dir.pwd
+TARGET_DIR = "#{CWD}/target"
+TARGET_LIB_DIR = "#{TARGET_DIR}/lib"
+TARGET_SRC_DIR = "#{TARGET_DIR}/src"
+TARGET_GEM_DIR = "#{TARGET_DIR}/gems/gems"
+TARGET_SPECS_DIR = "#{TARGET_DIR}/gems/specifications"
+TARGET_CLASSES_DIR = "#{TARGET_DIR}/classes"
+TARGET_DEPENDENCY_DIR = "#{TARGET_DIR}/dependency"
+TARGET_DEPENDENCY_UNPACKED_DIR = "#{TARGET_DIR}/dependency-unpacked"
+TARGET_CLUSTER_JAR = "#{TARGET_DIR}/cluster-topology.jar"
+
+REDSTORM_JAVA_SRC_DIR = "#{RedStorm::REDSTORM_HOME}/src/main"
+REDSTORM_LIB_DIR = "#{RedStorm::REDSTORM_HOME}/lib"
+
+SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples"
+DST_EXAMPLES = "#{CWD}/examples"
+
+SRC_IVY_DIR = "#{RedStorm::REDSTORM_HOME}/ivy"
+DST_IVY_DIR = "#{CWD}/ivy"
+CUSTOM_DEPENDENCIES = "#{CWD}/Dependencies"
+DEFAULT_IVY_SETTINGS = "#{SRC_IVY_DIR}/settings.xml"
+CUSTOM_IVY_SETTINGS = "#{DST_IVY_DIR}/settings.xml"
+
+
+module RedStorm
+
+  class Application
+    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake"
+
+    def self.local_storm_command(class_file, ruby_mode = nil)
+      src_dir = File.expand_path(File.dirname(class_file))
+      "java -Djruby.compat.version=#{RedStorm.jruby_mode_token(ruby_mode)} -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/storm/default/*:#{TARGET_DEPENDENCY_DIR}/topology/default/*:#{src_dir}/\" redstorm.TopologyLauncher local #{class_file}"
+    end
+
+    def self.cluster_storm_command(class_file, ruby_mode = nil)
+      "storm jar #{TARGET_CLUSTER_JAR} -Djruby.compat.version=#{RedStorm.jruby_mode_token(ruby_mode)} redstorm.TopologyLauncher cluster #{class_file}"
+    end
+
+    def self.usage
+      puts("usage: redstorm version")
+      puts("       redstorm install")
+      puts("       redstorm deps")
+      puts("       redstorm build")
+      puts("       redstorm examples")
+      puts("       redstorm bundle [BUNDLER_GROUP]")
+      puts("       redstorm jar DIR1, [DIR2, ...]")
+      puts("       redstorm local [--1.8|--1.9] TOPOLOGY_CLASS_PATH")
+      puts("       redstorm cluster [--1.8|--1.9] TOPOLOGY_CLASS_PATH")
+      exit(1)
+    end
+
+    def self.run(args)
+      if args.size > 0
+        if args[0] == "version"
+          puts("RedStorm v#{VERSION}")
+          exit
+        elsif ["install", "examples", "jar", "bundle", "deps", "build"].include?(args[0])
+          load(TASKS_FILE)
+          Rake::Task[args.shift].invoke(args.join(":"))
+          exit
+        elsif args.size >= 2 && ["local", "cluster"].include?(args[0])
+          env = args.delete_at(0)
+          version = args.delete("--1.8") || args.delete("--1.9")
+          if args.size == 1
+            file = args[0]
+            load(TASKS_FILE)
+            Rake::Task['launch'].invoke(env, version, file)
+            exit
+          end
+        end
+      end
+      usage
+    end
+
+    def self.subshell(command)
+      out = IO.popen(command, {:err => [:child, :out]}) {|io| io.read}
+      [!!$?.success?, out]
+    end
+
+  end
+
+end