redstorm 0.6.2 → 0.6.3

data/CHANGELOG.md

@@ -56,4 +56,10 @@
 # 0.6.2, 07-10-2012
 - issue #39, spout on_receive block will not be evaluated if :emit => false
 - issue #40, bolt fail method missing
-- integration tests support
+- integration tests support
+
+# 0.6.3, 10-09-2012
+- issue #28, allow specification of output_fields in topology DSL
+- issue #41, add support for ShellBolt and ShellSpout
+- issue #49, redstorm bundle not picking up default group in Gemfile
+- support constructor parameters for Java spout/bolt in topology DSL

data/README.md

@@ -1,4 +1,4 @@
-# RedStorm v0.6.2 - JRuby on Storm
+# RedStorm v0.6.3 - JRuby on Storm
 
 [![build status](https://secure.travis-ci.org/colinsurprenant/redstorm.png)](http://travis-ci.org/colinsurprenant/redstorm)
 
@@ -58,7 +58,7 @@ $ redstorm local|cluster [--1.8|--1.9] ...
 
   ``` ruby
   source :rubygems
-  gem "redstorm", "~> 0.6.2"
+  gem "redstorm", "~> 0.6.3"
   ```
 
 ## Usage overview
@@ -255,6 +255,25 @@ The [Storm wiki](https://github.com/nathanmarz/storm/wiki) has instructions on [
 
 [Ruby DSL Documentation](https://github.com/colinsurprenant/redstorm/wiki/Ruby-DSL-Documentation)
 
+## Multilang ShellSpout & ShellBolt support
+
+Please refer to [Using non JVM languages with Storm](https://github.com/nathanmarz/storm/wiki/Using-non-JVM-languages-with-Storm) for the complete information on Multilang & shelling in Storm.
+
+In RedStorm *ShellSpout* and *ShellBolt* are supported using the following construct in the topology definition:
+
+``` ruby
+bolt JRubyShellBolt, ["python", "splitsentence.py"] do
+  output_fields "word"
+  source SimpleSpout, :shuffle
+end
+```
+
+- `JRubyShellBolt` must be used for a *ShellBolt* and the array argument `["python", "splitsentence.py"]` are the arguments to the class constructor and are the *commands* to the *ShellBolt*.
+
+- The directory containing the topology class **must** contain a `resources/` directory with all the shell files.
+
+See the [shell topology example](https://github.com/colinsurprenant/redstorm/tree/master/examples/shell)
+
 ## RedStorm Development
 
 It is possible to fork the RedStorm project and run local and remote/cluster topologies directly from the project sources without installing the gem. This is a useful setup when contributing to the project.
@@ -307,10 +326,10 @@ Some ways you can contribute:
 
 If you want to list your RedStorm project here, contact me.
 
-- [Tweigeist](https://github.com/colinsurprenant/tweitgeist) - realtime computation of the top trending hashtags on Twitter. [Live Demo](http://tweitgeist.needium.com).
+- [Tweigeist](https://github.com/colinsurprenant/tweitgeist) - realtime computation of the top trending hashtags on Twitter. Live Demo in search of a new home.
 
 ## Author
-Colin Surprenant, [@colinsurprenant][twitter], [http://github.com/colinsurprenant][github], colin.surprenant@gmail.com, colin.surprenant@needium.com
+Colin Surprenant, [@colinsurprenant][twitter], [http://github.com/colinsurprenant][github], colin.surprenant@gmail.com
 
 ## Contributors
 Theo Hultberg, https://github.com/iconara

data/examples/shell/resources/splitsentence.py

@@ -0,0 +1,9 @@
+import storm
+
+class SplitSentenceBolt(storm.BasicBolt):
+    def process(self, tup):
+        words = tup.values[0].split(" ")
+        for word in words:
+          storm.emit([word])
+
+SplitSentenceBolt().run()

data/examples/shell/resources/storm.py

@@ -0,0 +1,206 @@
+import sys
+import os
+import traceback
+from collections import deque
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+json_encode = lambda x: json.dumps(x)
+json_decode = lambda x: json.loads(x)
+
+#reads lines and reconstructs newlines appropriately
+def readMsg():
+    msg = ""
+    while True:
+        line = sys.stdin.readline()[0:-1]
+        if line == "end":
+            break
+        msg = msg + line + "\n"
+    return json_decode(msg[0:-1])
+
+MODE = None
+ANCHOR_TUPLE = None
+
+#queue up commands we read while trying to read taskids
+pending_commands = deque()
+
+def readTaskIds():
+    if pending_taskids:
+        return pending_taskids.popleft()
+    else:
+        msg = readMsg()
+        while type(msg) is not list:
+            pending_commands.append(msg)
+            msg = readMsg()
+        return msg
+
+#queue up taskids we read while trying to read commands/tuples
+pending_taskids = deque()
+
+def readCommand():
+    if pending_commands:
+        return pending_commands.popleft()
+    else:
+        msg = readMsg()
+        while type(msg) is list:
+            pending_taskids.append(msg)
+            msg = readMsg()
+        return msg
+
+def readTuple():
+    cmd = readCommand()
+    return Tuple(cmd["id"], cmd["comp"], cmd["stream"], cmd["task"], cmd["tuple"])
+
+def sendMsgToParent(msg):
+    print json_encode(msg)
+    print "end"
+    sys.stdout.flush()
+
+def sync():
+    sendMsgToParent({'command':'sync'})
+
+def sendpid(heartbeatdir):
+    pid = os.getpid()
+    sendMsgToParent({'pid':pid})
+    open(heartbeatdir + "/" + str(pid), "w").close()    
+
+def emit(*args, **kwargs):
+    __emit(*args, **kwargs)
+    return readTaskIds()
+
+def emitDirect(task, *args, **kwargs):
+    kwargs[directTask] = task
+    __emit(*args, **kwargs)
+
+def __emit(*args, **kwargs):
+    global MODE
+    if MODE == Bolt:
+        emitBolt(*args, **kwargs)
+    elif MODE == Spout:
+        emitSpout(*args, **kwargs)
+
+def emitBolt(tup, stream=None, anchors = [], directTask=None):
+    global ANCHOR_TUPLE
+    if ANCHOR_TUPLE is not None:
+        anchors = [ANCHOR_TUPLE]
+    m = {"command": "emit"}
+    if stream is not None:
+        m["stream"] = stream
+    m["anchors"] = map(lambda a: a.id, anchors)
+    if directTask is not None:
+        m["task"] = directTask
+    m["tuple"] = tup
+    sendMsgToParent(m)
+    
+def emitSpout(tup, stream=None, id=None, directTask=None):
+    m = {"command": "emit"}
+    if id is not None:
+        m["id"] = id
+    if stream is not None:
+        m["stream"] = stream
+    if directTask is not None:
+        m["task"] = directTask
+    m["tuple"] = tup
+    sendMsgToParent(m)
+
+def ack(tup):
+    sendMsgToParent({"command": "ack", "id": tup.id})
+
+def fail(tup):
+    sendMsgToParent({"command": "fail", "id": tup.id})
+
+def log(msg):
+    sendMsgToParent({"command": "log", "msg": msg})
+
+def initComponent():
+    setupInfo = readMsg()
+    sendpid(setupInfo['pidDir'])
+    return [setupInfo['conf'], setupInfo['context']]
+
+class Tuple:
+    def __init__(self, id, component, stream, task, values):
+        self.id = id
+        self.component = component
+        self.stream = stream
+        self.task = task
+        self.values = values
+
+    def __repr__(self):
+        return '<%s%s>' % (
+                self.__class__.__name__,
+                ''.join(' %s=%r' % (k, self.__dict__[k]) for k in sorted(self.__dict__.keys())))
+
+class Bolt:
+    def initialize(self, stormconf, context):
+        pass
+
+    def process(self, tuple):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Bolt
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                tup = readTuple()
+                self.process(tup)
+        except Exception, e:
+            log(traceback.format_exc(e))
+
+class BasicBolt:
+    def initialize(self, stormconf, context):
+        pass
+
+    def process(self, tuple):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Bolt
+        global ANCHOR_TUPLE
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                tup = readTuple()
+                ANCHOR_TUPLE = tup
+                self.process(tup)
+                ack(tup)
+        except Exception, e:
+            log(traceback.format_exc(e))
+
+class Spout:
+    def initialize(self, conf, context):
+        pass
+
+    def ack(self, id):
+        pass
+
+    def fail(self, id):
+        pass
+
+    def nextTuple(self):
+        pass
+
+    def run(self):
+        global MODE
+        MODE = Spout
+        conf, context = initComponent()
+        self.initialize(conf, context)
+        try:
+            while True:
+                msg = readCommand()
+                if msg["command"] == "next":
+                    self.nextTuple()
+                if msg["command"] == "ack":
+                    self.ack(msg["id"])
+                if msg["command"] == "fail":
+                    self.fail(msg["id"])
+                sync()
+        except Exception, e:
+            log(traceback.format_exc(e))

data/examples/shell/shell_topology.rb

@@ -0,0 +1,41 @@
+require 'red_storm'
+require 'thread'
+
+java_import 'redstorm.storm.jruby.JRubyShellBolt'
+
+class SimpleSpout < RedStorm::SimpleSpout
+  on_init do
+    @q = Queue.new
+    @q << "the quick red fox"
+  end
+
+  on_send do
+    # avoid putting the thread to sleep endlessly on @q.pop which will prevent local cluster.shutdown
+    sleep(1)
+    @q.pop unless @q.empty?
+  end
+end
+
+class ShellTopology < RedStorm::SimpleTopology
+  spout SimpleSpout do
+    output_fields :string
+  end
+
+  bolt JRubyShellBolt, ["python", "splitsentence.py"] do
+    output_fields "word"
+    source SimpleSpout, :shuffle
+  end
+
+  configure do |env|
+    debug true
+  end
+
+  on_submit do |env|
+    case env
+    when :local
+      sleep(10)
+      cluster.shutdown
+    end
+  end
+end
+

data/lib/red_storm/application.rb

@@ -25,7 +25,8 @@ module RedStorm
     TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
 
     def self.local_storm_command(class_file, ruby_mode = nil)
-      "java -Djruby.compat.version=#{RedStorm.jruby_mode_token(ruby_mode)} -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\" redstorm.TopologyLauncher local #{class_file}"
+      src_dir = File.expand_path(File.dirname(class_file))
+      "java -Djruby.compat.version=#{RedStorm.jruby_mode_token(ruby_mode)} -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*:#{src_dir}/\" redstorm.TopologyLauncher local #{class_file}"
     end
 
     def self.cluster_storm_command(class_file, ruby_mode = nil)
@@ -67,6 +68,12 @@ module RedStorm
       end
       usage
     end
+
+    def self.subshell(command)
+      out = IO.popen([command, {:err => [:child, :out]}]) {|io| io.read}
+      [!!$?.success?, out]
+    end
+
   end
 
 end

data/lib/red_storm/simple_topology.rb

@@ -2,6 +2,7 @@ require 'java'
 require 'red_storm/configuration'
 require 'red_storm/configurator'
 
+
 module RedStorm
 
   class TopologyDefinitionError < StandardError; end
@@ -13,14 +14,20 @@ module RedStorm
     DEFAULT_BOLT_PARALLELISM = 1
 
     class ComponentDefinition < Configurator
-      attr_reader :clazz, :parallelism
+      attr_reader :clazz, :constructor_args, :parallelism
       attr_accessor :id # ids are forced to string
 
-      def initialize(component_class, id, parallelism)
+      def initialize(component_class, constructor_args, id, parallelism)
         super()
         @clazz = component_class
+        @constructor_args = constructor_args
         @id = id.to_s
         @parallelism = parallelism
+        @output_fields = []
+      end
+
+      def output_fields(*args)
+        args.empty? ? @output_fields : @output_fields = args.map(&:to_s)
       end
 
       def is_java?
@@ -29,13 +36,22 @@ module RedStorm
     end
 
     class SpoutDefinition < ComponentDefinition
+      
+      # WARNING non-dry see BoltDefinition#new_instance
       def new_instance(base_class_path)
-        is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
+        if @clazz.name == "Java::RedstormStormJruby::JRubyShellSpout"
+          @clazz.new(constructor_args, @output_fields)
+        elsif is_java?
+          @clazz.new(*constructor_args)
+        else
+          JRubySpout.new(base_class_path, @clazz.name, @output_fields)
+        end
+        # is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
       end
     end
           
     class BoltDefinition < ComponentDefinition
-      attr_accessor :sources
+      attr_accessor :sources, :command
 
       def initialize(*args)
         super
@@ -72,7 +88,15 @@ module RedStorm
       end
 
       def new_instance(base_class_path)
-        is_java? ? @clazz.new : JRubyBolt.new(base_class_path, @clazz.name)
+        # WARNING non-dry see BoltDefinition#new_instance
+        if @clazz.name == "Java::RedstormStormJruby::JRubyShellBolt"
+          @clazz.new(constructor_args, @output_fields)
+        elsif is_java?
+          @clazz.new(*constructor_args)
+        else
+          JRubyBolt.new(base_class_path, @clazz.name, @output_fields)
+        end
+        # is_java? ? @clazz.new : @clazz.is_a?(SimpleBolt) ? JRubyBolt.new(base_class_path, @clazz.name) : @clazz.new
       end
     end
 
@@ -80,16 +104,24 @@ module RedStorm
       @log ||= org.apache.log4j.Logger.getLogger(self.name)
     end
 
-    def self.spout(spout_class, options = {}, &spout_block)
+    # def self.spout(spout_class, contructor_args = [], options = {}, &spout_block)
+    def self.spout(spout_class, *args, &spout_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
       spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
-      spout = SpoutDefinition.new(spout_class, spout_options[:id], spout_options[:parallelism])
+
+      spout = SpoutDefinition.new(spout_class, contructor_args, spout_options[:id], spout_options[:parallelism])
       spout.instance_exec(&spout_block) if block_given?
       self.components << spout
     end
 
-    def self.bolt(bolt_class, options = {}, &bolt_block)
+    # def self.bolt(bolt_class, contructor_args = [], options = {}, &bolt_block)
+    def self.bolt(bolt_class, *args, &bolt_block)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      contructor_args = !args.empty? ? args.pop : []
       bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
-      bolt = BoltDefinition.new(bolt_class, bolt_options[:id], bolt_options[:parallelism])
+
+      bolt = BoltDefinition.new(bolt_class, contructor_args, bolt_options[:id], bolt_options[:parallelism])
       raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
       bolt.instance_exec(&bolt_block)
       self.components << bolt

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.6.2'
+  VERSION = '0.6.3'
 end

data/lib/tasks/red_storm.rake

@@ -88,8 +88,22 @@ task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
       exclude :name => "tasks/**"
     end
     if args[:include_dir]
+      dirs = args[:include_dir].split(":")
+
+      # first add any resources/ dir in the tree in the jar root - requirement for ShellBolt multilang resources
+      dirs.each do |dir|
+        resources_dirs = Dir.glob("#{dir}/**/resources")
+        resources_dirs.each do |resources_dir|
+          resources_parent = resources_dir.gsub("/resources", "")
+          fileset :dir => resources_parent do
+            include :name => "resources/**/*"
+          end
+        end
+      end
+
+      # include complete source dir tree (note we don't care about potential duplicated resources dir)
       fileset :dir => CWD do
-        args[:include_dir].split(":").each{|dir| include :name => "#{dir}/**/*"}
+        dirs.each{|dir| include :name => "#{dir}/**/*"}
       end
     end
     manifest do
@@ -157,8 +171,8 @@ end
 
 task :bundle, [:groups] => :setup do |t, args|
   require 'bundler'
-  args.with_defaults(:groups => 'default')
-  groups = args[:groups].split(':').map(&:to_sym)
+  defaulted_args = {:groups => 'default'}.merge(args.to_hash.delete_if{|k, v| v.to_s.empty?})
+  groups = defaulted_args[:groups].split(':').map(&:to_sym)
   Bundler.definition.specs_for(groups).each do |spec|
     unless spec.full_name =~ /^bundler-\d+/
       destination_path = "#{TARGET_GEM_DIR}/#{spec.full_name}"

data/src/main/redstorm/storm/jruby/JRubyBolt.java

@@ -5,6 +5,7 @@ import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.IRichBolt;
 import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
 import java.util.Map;
 
 /**
@@ -20,16 +21,19 @@ import java.util.Map;
 public class JRubyBolt implements IRichBolt {
   IRichBolt _proxyBolt;
   String _realBoltClassName;
-  String _baseClassPath; 
+  String _baseClassPath;
+  String[] _fields;
+
   /**
    * create a new JRubyBolt
    * 
    * @param baseClassPath the topology/project base JRuby class file path 
    * @param realBoltClassName the fully qualified JRuby bolt implementation class name
    */
-  public JRubyBolt(String baseClassPath, String realBoltClassName) {
+  public JRubyBolt(String baseClassPath, String realBoltClassName, String[] fields) {
     _baseClassPath = baseClassPath;
     _realBoltClassName = realBoltClassName;
+    _fields = fields;
   }
 
   @Override
@@ -54,8 +58,12 @@ public class JRubyBolt implements IRichBolt {
     // declareOutputFields is executed in the topology creation time, before serialisation.
     // do not set the _proxyBolt instance variable here to avoid JRuby serialization
     // issues. Just create tmp bolt instance to call declareOutputFields.
-    IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
-    bolt.declareOutputFields(declarer);
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+      bolt.declareOutputFields(declarer);
+    }
   }
 
   @Override

data/src/main/redstorm/storm/jruby/JRubyShellBolt.java

@@ -0,0 +1,26 @@
+package redstorm.storm.jruby;
+
+import backtype.storm.task.ShellBolt;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+    
+public class JRubyShellBolt  extends ShellBolt  implements IRichBolt {
+  private String[] _fields;
+
+  public JRubyShellBolt(String[] command, String[] fields) {
+    super(command);
+    _fields = fields;
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declare(new Fields(_fields));
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    return null;
+  }
+}

data/src/main/redstorm/storm/jruby/JRubyShellSpout.java

@@ -0,0 +1,26 @@
+package backtype.storm.clojure;
+
+import backtype.storm.spout.ShellSpout;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import java.util.Map;
+
+public class JRubyShellSpout extends ShellSpout implements IRichSpout {
+  private String[] _fields;
+
+  public JRubyShellSpout(String[] command, String[] fields) {
+    super(command);
+    _fields = fields;
+  }
+
+  @Override
+  public void declareOutputFields(OutputFieldsDeclarer declarer) {
+    declarer.declare(new Fields(_fields));
+  }
+
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    return null;
+  }
+}

data/src/main/redstorm/storm/jruby/JRubySpout.java

@@ -5,6 +5,7 @@ import backtype.storm.task.TopologyContext;
 import backtype.storm.topology.IRichSpout;
 import backtype.storm.topology.OutputFieldsDeclarer;
 import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Fields;
 import java.util.Map;
 
 /**
@@ -21,16 +22,18 @@ public class JRubySpout implements IRichSpout {
   IRichSpout _proxySpout;
   String _realSpoutClassName;
   String _baseClassPath;
-  
+  String[] _fields;
+
   /**
    * create a new JRubySpout
    * 
    * @param baseClassPath the topology/project base JRuby class file path 
    * @param realSpoutClassName the fully qualified JRuby spout implementation class name
    */
-  public JRubySpout(String baseClassPath, String realSpoutClassName) {
+  public JRubySpout(String baseClassPath, String realSpoutClassName, String[] fields) {
     _baseClassPath = baseClassPath;
     _realSpoutClassName = realSpoutClassName;
+    _fields = fields;
   }
 
   @Override
@@ -75,8 +78,12 @@ public class JRubySpout implements IRichSpout {
     // declareOutputFields is executed in the topology creation time before serialisation.
     // do not set the _proxySpout instance variable here to avoid JRuby serialization
     // issues. Just create tmp spout instance to call declareOutputFields.
-    IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    spout.declareOutputFields(declarer);
+    if (_fields.length > 0) {
+      declarer.declare(new Fields(_fields));
+    } else {
+      IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+      spout.declareOutputFields(declarer);
+    }
   }  
 
   @Override

metadata

@@ -2,14 +2,14 @@
 name: redstorm
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Colin Surprenant
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-07-10 00:00:00.000000000 Z
+date: 2012-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -91,6 +91,9 @@ files:
 - examples/native/random_sentence_spout.rb
 - examples/native/split_sentence_bolt.rb
 - examples/native/word_count_bolt.rb
+- examples/shell/shell_topology.rb
+- examples/shell/resources/splitsentence.py
+- examples/shell/resources/storm.py
 - examples/simple/exclamation_bolt.rb
 - examples/simple/exclamation_topology.rb
 - examples/simple/exclamation_topology2.rb
@@ -102,6 +105,8 @@ files:
 - examples/simple/word_count_bolt.rb
 - examples/simple/word_count_topology.rb
 - src/main/redstorm/storm/jruby/JRubyBolt.java
+- src/main/redstorm/storm/jruby/JRubyShellBolt.java
+- src/main/redstorm/storm/jruby/JRubyShellSpout.java
 - src/main/redstorm/storm/jruby/JRubySpout.java
 - bin/redstorm
 - Rakefile