kb-redstorm 0.6.4

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rspec/core/rake_task'
+
+
+RSpec::Core::RakeTask.new(:spec) do
+  system("ruby -v")
+  module RedStorm; SPECS_CONTEXT = true; end
+end
+
+task :default => :spec
+
+load 'lib/tasks/red_storm.rake'

data/bin/redstorm

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'rubygems' 
+require 'rake'
+
+begin
+  # will work from gem, since lib dir is in gem require_paths
+  require 'red_storm/application'
+rescue LoadError
+  # will work within RedStorm dev project
+  $:.unshift './lib'
+  require 'red_storm/application'
+end
+
+RedStorm::Application.run(ARGV.dup)

data/examples/native/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gem 'redis'

data/examples/native/cluster_word_count_topology.rb

@@ -0,0 +1,25 @@
+require 'red_storm'
+require 'examples/native/random_sentence_spout'
+require 'examples/native/split_sentence_bolt'
+require 'examples/native/word_count_bolt'
+
+module RedStorm
+  module Examples
+    class ClusterWordCountTopology
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
+        builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 4).shuffleGrouping('RandomSentenceSpout')
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 4).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+
+        conf = Backtype::Config.new
+        conf.setDebug(true)
+        conf.setNumWorkers(20);
+        conf.setMaxSpoutPending(1000);
+        StormSubmitter.submitTopology("word_count", conf, builder.createTopology);
+      end
+    end
+  end
+end

data/examples/native/exclamation_bolt.rb

@@ -0,0 +1,21 @@
+module RedStorm
+  module Examples
+    class ExclamationBolt
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
+
+      def execute(tuple)
+        @collector.emit(tuple, Values.new(tuple.getString(0) + "!!!"))
+        @collector.ack(tuple)
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
+  end
+end

data/examples/native/local_exclamation_topology.rb

@@ -0,0 +1,31 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'lib/red_storm'
+require 'examples/native/exclamation_bolt'
+
+# this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+
+module RedStorm
+  module Examples
+    class LocalExclamationTopology
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
+        builder.setBolt('ExclamationBolt1', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt2', JRubyBolt.new(base_class_path, 'RedStorm::Examples::ExclamationBolt'), 3).shuffleGrouping('ExclamationBolt1')
+                
+        conf = Backtype::Config.new
+        conf.setDebug(true)
+        
+        cluster = LocalCluster.new
+        cluster.submitTopology("exclamation", conf, builder.createTopology)
+        sleep(5)
+        cluster.killTopology("exclamation")
+        cluster.shutdown
+      end
+    end
+  end
+end

data/examples/native/local_exclamation_topology2.rb

@@ -0,0 +1,48 @@
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'lib/red_storm'
+
+module RedStorm
+  module Examples
+    class ExclamationBolt2
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
+
+      def execute(tuple)
+        @collector.emit(tuple, Values.new("!#{tuple.getString(0)}!"))
+        @collector.ack(tuple)
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
+
+    # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
+
+    class LocalExclamationTopology2
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        
+        builder.setSpout('TestWordSpout', TestWordSpout.new, 10)     
+        builder.setBolt('ExclamationBolt21', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 3).shuffleGrouping('TestWordSpout')
+        builder.setBolt('ExclamationBolt22', JRubyBolt.new(base_class_path, "RedStorm::Examples::ExclamationBolt2"), 2).shuffleGrouping('ExclamationBolt21')
+                
+        conf = Backtype::Config.new
+        conf.setDebug(true)
+        
+        cluster = LocalCluster.new
+        cluster.submitTopology("exclamation", conf, builder.createTopology)
+        sleep(5)
+        cluster.killTopology("exclamation")
+        cluster.shutdown
+      end
+    end
+  end
+end

data/examples/native/local_redis_word_count_topology.rb

@@ -0,0 +1,69 @@
+require 'bundler/setup'
+require 'redis'
+require 'thread'
+require 'lib/red_storm'
+require 'examples/native/word_count_bolt'
+
+module RedStorm
+  module Examples
+    # RedisWordSpout reads the Redis queue "test" on localhost:6379 
+    # and emits each word items pop'ed from the queue.
+    class RedisWordSpout
+      def open(conf, context, collector)
+        @collector = collector
+        @q = Queue.new
+        @redis_reader = detach_redis_reader
+      end
+      
+      def next_tuple
+        # per doc nextTuple should not block, and sleep a bit when there's no data to process.
+        if @q.size > 0
+          @collector.emit(Values.new(@q.pop))
+        else
+          sleep(0.1)
+        end
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+
+      private
+
+      def detach_redis_reader
+        Thread.new do
+          Thread.current.abort_on_exception = true
+
+          redis = Redis.new(:host => "localhost", :port => 6379)
+          loop do
+            if data = redis.blpop("test", 0)
+              @q << data[1]
+            end
+          end
+        end
+      end
+    end
+
+    class LocalRedisWordCountTopology
+      RedStorm::Configuration.topology_class = self
+
+      def start(base_class_path, env)
+        builder = TopologyBuilder.new
+        builder.setSpout('RedisWordSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RedisWordSpout"), 1)
+        builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 3).fieldsGrouping('RedisWordSpout', Fields.new("word"))
+
+        conf = Backtype::Config.new
+        conf.setDebug(true)
+        conf.setMaxTaskParallelism(3)
+
+        cluster = LocalCluster.new
+        cluster.submitTopology("redis_word_count", conf, builder.createTopology)
+        sleep(600)
+        cluster.shutdown
+      end
+    end
+  end
+end

data/examples/native/local_word_count_topology.rb

@@ -0,0 +1,27 @@
+require 'lib/red_storm'
+require 'examples/native/random_sentence_spout'
+require 'examples/native/split_sentence_bolt'
+require 'examples/native/word_count_bolt'
+
+
+module Examples
+  class LocalWordCountTopology
+    RedStorm::Configuration.topology_class = self
+
+    def start(base_class_path, env)
+      builder = TopologyBuilder.new
+      builder.setSpout('RandomSentenceSpout', JRubySpout.new(base_class_path, "RedStorm::Examples::RandomSentenceSpout"), 5)
+      builder.setBolt('SplitSentenceBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::SplitSentenceBolt"), 8).shuffleGrouping('RandomSentenceSpout')
+      builder.setBolt('WordCountBolt', JRubyBolt.new(base_class_path, "RedStorm::Examples::WordCountBolt"), 12).fieldsGrouping('SplitSentenceBolt', Fields.new("word"))
+
+      conf = Backtype::Config.new
+      conf.setDebug(true)
+      conf.setMaxTaskParallelism(3)
+
+      cluster = LocalCluster.new
+      cluster.submitTopology("word_count", conf, builder.createTopology)
+      sleep(5)
+      cluster.shutdown
+    end
+  end
+end

data/examples/native/random_sentence_spout.rb

@@ -0,0 +1,30 @@
+module RedStorm
+  module Examples
+    class RandomSentenceSpout
+      def initialize
+        @sentences = [
+          "the cow jumped over the moon",
+          "an apple a day keeps the doctor away",
+          "four score and seven years ago",
+          "snow white and the seven dwarfs",
+          "i am at two with nature"
+        ]
+      end 
+
+      def open(conf, context, collector)
+        @collector = collector
+      end
+      
+      def next_tuple
+        @collector.emit(Values.new(@sentences[rand(@sentences.length)]))
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
+  end
+end

data/examples/native/split_sentence_bolt.rb

@@ -0,0 +1,20 @@
+module RedStorm
+  module Examples
+    class SplitSentenceBolt
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
+
+      def execute(tuple)
+        tuple.getString(0).split(" ").each {|w| @collector.emit(Values.new(w)) }
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word"))
+      end
+    end
+  end
+end

data/examples/native/word_count_bolt.rb

@@ -0,0 +1,26 @@
+module RedStorm
+  module Examples
+    class WordCountBolt
+      def initialize
+        @counts = Hash.new{|h, k| h[k] = 0}
+      end
+
+      def prepare(conf, context, collector)
+        @collector = collector
+      end
+
+      def execute(tuple)
+        word = tuple.getString(0)
+        @counts[word] += 1
+        @collector.emit(Values.new(word, @counts[word]))
+      end
+
+      def get_component_configuration
+      end
+
+      def declare_output_fields(declarer)
+        declarer.declare(Fields.new("word", "count"))
+      end
+    end
+  end
+end

data/examples/shell/resources/splitsentence.py

@@ -0,0 +1,9 @@
+import storm
+
+class SplitSentenceBolt(storm.BasicBolt):
+    def process(self, tup):
+        words = tup.values[0].split(" ")
+        for word in words:
+          storm.emit([word])
+
+SplitSentenceBolt().run()

data/examples/shell/resources/storm.py

@@ -0,0 +1,206 @@
+import sys
+import os
+import traceback
+from collections import deque
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+json_encode = lambda x: json.dumps(x)
+json_decode = lambda x: json.loads(x)
+
+#reads lines and reconstructs newlines appropriately
+def readMsg():
+    msg = ""
+    while True:
+        line = sys.stdin.readline()[0:-1]
+        if line == "end":
+            break
+        msg = msg + line + "\n"
+    return json_decode(msg[0:-1])
+
+MODE = None
+ANCHOR_TUPLE = None
+
+#queue up commands we read while trying to read taskids
+pending_commands = deque()
+
+def readTaskIds():
+    if pending_taskids:
+        return pending_taskids.popleft()
+    else:
+        msg = readMsg()
+        while type(msg) is not list:
+            pending_commands.append(msg)
+            msg = readMsg()
+        return msg
+
+#queue up taskids we read while trying to read commands/tuples
+pending_taskids = deque()
+
+def readCommand():
+    if pending_commands:
+        return pending_commands.popleft()
+    else:
+        msg = readMsg()
+        while type(msg) is list:
+            pending_taskids.append(msg)
+            msg = readMsg()
+        return msg
+
+def readTuple():
+    cmd = readCommand()
+    return Tuple(cmd["id"], cmd["comp"], cmd["stream"], cmd["task"], cmd["tuple"])
+
+def sendMsgToParent(msg):
+    print json_encode(msg)
+    print "end"
+    sys.stdout.flush()
+
+def sync():
+    sendMsgToParent({'command':'sync'})
+
+def sendpid(heartbeatdir):
+    pid = os.getpid()
+    sendMsgToParent({'pid':pid})
+    open(heartbeatdir + "/" + str(pid), "w").close()    
+
+def emit(*args, **kwargs):
+    __emit(*args, **kwargs)
+    return readTaskIds()
+
+def emitDirect(task, *args, **kwargs):
+    kwargs[directTask] = task
+    __emit(*args, **kwargs)
+
+def __emit(*args, **kwargs):
+    global MODE
+    if MODE == Bolt:
+        emitBolt(*args, **kwargs)
+    elif MODE == Spout:
+        emitSpout(*args, **kwargs)
+
+def emitBolt(tup, stream=None, anchors = [], directTask=None):
+    global ANCHOR_TUPLE
+    if ANCHOR_TUPLE is not None:
+        anchors = [ANCHOR_TUPLE]
+    m = {"command": "emit"}
+    if stream is not None:
+        m["stream"] = stream
+    m["anchors"] = map(lambda a: a.id, anchors)
+    if directTask is not None:
+        m["task"] = directTask
+    m["tuple"] = tup
+    sendMsgToParent(m)
+    
+def emitSpout(tup, stream=None, id=None, directTask=None):
+    m = {"command": "emit"}
+    if id is not None:
+        m["id"] = id
+    if stream is not None:
+        m["stream"] = stream
+    if directTask is not None:
+        m["task"] = directTask
+    m["tuple"] = tup
+    sendMsgToParent(m)
+
+def ack(tup):
+    sendMsgToParent({"command": "ack", "id": tup.id})
+
+def fail(tup):
+    sendMsgToParent({"command": "fail", "id": tup.id})
+
+def log(msg):
+    sendMsgToParent({"command": "log", "msg": msg})
+
+def initComponent():
+    setupInfo = readMsg()
+    sendpid(setupInfo['pidDir'])
+    return [setupInfo['conf'], setupInfo['context']]
+
+class Tuple:
+    def __init__(self, id, component, stream, task, values):
+        self.id = id
+        self.component = component
+        self.stream = stream
+        self.task = task
+        self.values = values
+
+    def __repr__(self):
+        return '<%s%s>' % (
+                self.__class__.__name__,
+                ''.join(' %s=%r' % (k, self.__dict__[k]) for k in sorted(self.__dict__.keys())))
+
+class Bolt:
+    def initialize(self, stormconf, context):
+        pass
+
+    def process(self, tuple):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Bolt
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                tup = readTuple()
+                self.process(tup)
+        except Exception, e:
+            log(traceback.format_exc(e))
+
+class BasicBolt:
+    def initialize(self, stormconf, context):
+        pass
+
+    def process(self, tuple):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Bolt
+        global ANCHOR_TUPLE
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                tup = readTuple()
+                ANCHOR_TUPLE = tup
+                self.process(tup)
+                ack(tup)
+        except Exception, e:
+            log(traceback.format_exc(e))
+
+class Spout:
+    def initialize(self, conf, context):
+        pass
+
+    def ack(self, id):
+        pass
+
+    def fail(self, id):
+        pass
+
+    def nextTuple(self):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Spout
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                msg = readCommand()
+                if msg["command"] == "next":
+                    self.nextTuple()
+                if msg["command"] == "ack":
+                    self.ack(msg["id"])
+                if msg["command"] == "fail":
+                    self.fail(msg["id"])
+                sync()
+        except Exception, e:
+            log(traceback.format_exc(e))