kb-redstorm 0.6.5 → 0.6.6

data/examples/simple/exclamation_topology.rb

@@ -1,36 +1,32 @@
 java_import 'backtype.storm.testing.TestWordSpout'
 
+require 'red_storm'
 require 'examples/simple/exclamation_bolt'
 
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 
 module RedStorm
   module Examples
-    class ExclamationTopology < RedStorm::SimpleTopology
-      spout TestWordSpout, :parallelism => 5 do
+    class ExclamationTopology < SimpleTopology
+      spout TestWordSpout, :parallelism => 2 do
         debug true
       end
       
       bolt ExclamationBolt, :parallelism => 2 do
         source TestWordSpout, :shuffle
-        # max_task_parallelism 1
       end
       
       bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
         source ExclamationBolt, :shuffle
-        # max_task_parallelism 1
         debug true
       end
 
       configure do |env|
         debug false
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 40
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 4
+          max_spout_pending(1000)
         end
       end
 

data/examples/simple/exclamation_topology2.rb

@@ -1,9 +1,10 @@
-java_import 'backtype.storm.testing.TestWordSpout'
-require 'red_storm'
-
 # this example topology uses the Storm TestWordSpout and our own JRuby ExclamationBolt
 # and a locally defined ExclamationBolt
 
+java_import 'backtype.storm.testing.TestWordSpout'
+
+require 'red_storm'
+
 module RedStorm
   module Examples
     class ExclamationBolt < RedStorm::SimpleBolt
@@ -12,9 +13,9 @@ module RedStorm
     end
 
     class ExclamationTopology2 < RedStorm::SimpleTopology
-      spout TestWordSpout, :parallelism => 10
+      spout TestWordSpout, :parallelism => 2
       
-      bolt ExclamationBolt, :parallelism => 3 do
+      bolt ExclamationBolt, :parallelism => 2 do
         source TestWordSpout, :shuffle
       end
       
@@ -24,13 +25,10 @@ module RedStorm
 
       configure do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 3
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 4
+          max_spout_pending(1000)
         end
       end
 

data/examples/simple/hello_world_topology.rb

@@ -0,0 +1,22 @@
+require 'red_storm'
+
+class HelloWorldSpout < RedStorm::SimpleSpout
+  on_init {@words = ["hello", "world"]}
+  on_send {@words.shift unless @words.empty?}
+end
+
+class HelloWorldBolt < RedStorm::SimpleBolt
+  on_receive :emit => false do |tuple|
+    log.info(tuple.getString(0))
+  end
+end
+
+class HelloWorldTopology < RedStorm::SimpleTopology
+  spout HelloWorldSpout do
+    output_fields :word
+  end
+
+  bolt HelloWorldBolt do
+    source HelloWorldSpout, :global
+  end
+end

data/examples/simple/kafka_topology.rb

@@ -1,26 +1,26 @@
-require 'red_storm'
 java_import 'storm.kafka.KafkaConfig'
 java_import 'storm.kafka.SpoutConfig'
 java_import 'storm.kafka.StringScheme'
 java_import 'storm.kafka.KafkaSpout'
 
+require 'red_storm'
+
 # the KafkaTopology obviously requires a Kafka server running, you can ajust the
-# host and port below. 
+# host and port below.
 #
 # custom dependencies are required for the Kafka and Scala jars. put the following
-# dependencies in the "Dependencies" file in the root of your RedStorm project:
+# dependencies in the "ivy/topology_dependencies.xml" file in the root of your RedStorm project:
 #
-# {
-#   :storm_artifacts => [
-#     "storm:storm:0.8.1, transitive=true",
-#   ],
-#   :topology_artifacts => [
-#     "org.jruby:jruby-complete:1.6.8, transitive=false",
-#     "org.scala-lang:scala-library:2.8.0, transitive=false",
-#     "storm:kafka:0.7.0-incubating, transitive=false",
-#     "storm:storm-kafka:0.8.0-wip4, transitive=false",
-#   ],
-# }
+# <?xml version="1.0"?>
+# <ivy-module version="2.0">
+#   <info organisation="redstorm" module="topology-deps"/>
+#   <dependencies>
+#     <dependency org="org.jruby" name="jruby-core" rev="1.7.3" conf="default" transitive="true"/>
+#     <dependency org="org.scala-lang" name="scala-library" rev="2.8.0" conf="default" transitive="false"/>
+#     <dependency org="storm" name="kafka" rev="0.7.0-incubating" conf="default" transitive="false"/>
+#     <dependency org="storm" name="storm-kafka" rev="0.8.0-wip4" conf="default" transitive="false"/>
+#   </dependencies>
+# </ivy-module>
 
 class KafkaTopology < RedStorm::SimpleTopology
   spout_config = SpoutConfig.new(
@@ -36,7 +36,7 @@ class KafkaTopology < RedStorm::SimpleTopology
   end
 
   spout KafkaSpout, [spout_config]
-      
+
   bolt SplitStringBolt do
     output_fields :word
     source KafkaSpout, :shuffle

data/examples/simple/redis_word_count_topology.rb

@@ -1,9 +1,7 @@
-require 'rubygems'
 require 'red_storm'
-
+require 'examples/simple/word_count_bolt'
 require 'redis'
 require 'thread'
-require 'examples/simple/word_count_bolt'
 
 module RedStorm
   module Examples
@@ -46,14 +44,14 @@ module RedStorm
 
       configure do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
+        # set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
         case env
         when :local
           max_task_parallelism 3
         when :cluster
           max_task_parallelism 5
           num_workers 20
-          max_spout_pending(1000);
+          max_spout_pending(1000)
         end
       end
     end

data/examples/simple/ruby_version_topology.rb

@@ -6,7 +6,13 @@ module RedStorm
   module Examples
     class VersionSpout < RedStorm::SimpleSpout
       output_fields :dummy
-      on_init {log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")}
+      on_init do
+        log.info("***************** RUBY_VERSION=#{RUBY_VERSION}")
+        log.info("***************** JRUBY_VERSION=#{JRUBY_VERSION}")
+        log.info("***************** VERSION=#{VERSION}")
+        log.info("***************** RUBY_ENGINE=#{RUBY_ENGINE}")
+        log.info("***************** RUBY_PLATFORM=#{RUBY_PLATFORM}")
+      end
       on_send {}
     end
 

data/examples/simple/word_count_topology.rb

@@ -1,3 +1,4 @@
+require 'red_storm'
 require 'examples/simple/random_sentence_spout'
 require 'examples/simple/split_sentence_bolt'
 require 'examples/simple/word_count_bolt'
@@ -5,25 +6,22 @@ require 'examples/simple/word_count_bolt'
 module RedStorm
   module Examples
     class WordCountTopology < SimpleTopology
-      spout RandomSentenceSpout, :parallelism => 5
+      spout RandomSentenceSpout, :parallelism => 2
       
-      bolt SplitSentenceBolt, :parallelism => 8 do
+      bolt SplitSentenceBolt, :parallelism => 2 do
         source RandomSentenceSpout, :shuffle
       end
       
-      bolt WordCountBolt, :parallelism => 12 do
+      bolt WordCountBolt, :parallelism => 2 do
         source SplitSentenceBolt, :fields => ["word"]
       end
 
       configure :word_count do |env|
         debug true
-        set "topology.worker.childopts", "-Djruby.compat.version=RUBY1_9"
-        case env
-        when :local
-          max_task_parallelism 3
-        when :cluster
-          num_workers 20
-          max_spout_pending(1000);
+        max_task_parallelism 4
+        if env == :cluster
+          num_workers 6
+          max_spout_pending(1000)
         end
       end
 

data/ivy/settings.xml

@@ -1,3 +1,4 @@
+<?xml version="1.0"?>
 <ivysettings>
   <settings defaultResolver="repositories"/>
   <resolvers>

data/ivy/storm_dependencies.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<ivy-module version="2.0">
+  <info organisation="redstorm" module="storm-deps"/>
+  <dependencies>
+    <dependency org="storm" name="storm" rev="0.8.2" conf="default" transitive="true" />
+    <override org="org.slf4j" module="slf4j-log4j12" rev="1.6.3"/>
+  </dependencies>
+</ivy-module>

data/ivy/topology_dependencies.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<ivy-module version="2.0">
+  <info organisation="redstorm" module="topology-deps"/>
+  <dependencies>
+    <dependency org="org.jruby" name="jruby-core" rev="1.7.3" conf="default" transitive="true"/>
+  </dependencies>
+</ivy-module>

data/lib/red_storm/application.rb

@@ -20,15 +20,17 @@ DST_EXAMPLES = "#{CWD}/examples"
 
 SRC_IVY_DIR = "#{RedStorm::REDSTORM_HOME}/ivy"
 DST_IVY_DIR = "#{CWD}/ivy"
-CUSTOM_DEPENDENCIES = "#{CWD}/Dependencies"
 DEFAULT_IVY_SETTINGS = "#{SRC_IVY_DIR}/settings.xml"
 CUSTOM_IVY_SETTINGS = "#{DST_IVY_DIR}/settings.xml"
-
+DEFAULT_IVY_STORM_DEPENDENCIES = "#{SRC_IVY_DIR}/storm_dependencies.xml"
+CUSTOM_IVY_STORM_DEPENDENCIES = "#{DST_IVY_DIR}/storm_dependencies.xml"
+DEFAULT_IVY_TOPOLOGY_DEPENDENCIES = "#{SRC_IVY_DIR}/topology_dependencies.xml"
+CUSTOM_IVY_TOPOLOGY_DEPENDENCIES = "#{DST_IVY_DIR}/topology_dependencies.xml"
 
 module RedStorm
 
-  class Application
-    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake"
+  class Application 
+    TASKS_FILE = "#{RedStorm::REDSTORM_HOME}/lib/tasks/red_storm.rake" 
 
     def self.local_storm_command(class_file, ruby_mode = nil)
       src_dir = File.expand_path(File.dirname(class_file))
@@ -76,7 +78,7 @@ module RedStorm
     end
 
     def self.subshell(command)
-      out = IO.popen(command, {:err => [:child, :out]}) {|io| io.read}
+      out = IO.popen(command, STDERR => STDOUT) {|io| io.read}
       [!!$?.success?, out]
     end
 

data/lib/red_storm/configurator.rb

@@ -20,7 +20,7 @@ module RedStorm
     private
 
     def self.camel_case(s)
-      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_|\.)(.)/) { $1.upcase }
     end
   end
 end

data/lib/red_storm/proxy/batch_bolt.rb

@@ -0,0 +1,63 @@
+require 'java'
+
+java_import 'backtype.storm.coordination.BatchOutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.topology.IRichBolt'
+java_import 'backtype.storm.coordination.IBatchBolt'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+# the Bolt class is a proxy to the real bolt to avoid having to deal with all the
+# Java artifacts when creating a bolt.
+#
+# The real bolt class implementation must define these methods:
+# - prepare(conf, context, collector)
+# - execute(tuple)
+# - declare_output_fields
+#
+# and optionnaly:
+# - cleanup
+#
+class BatchBolt
+  java_implements IBatchBolt
+
+  java_signature 'IBatchBolt (String base_class_path, String real_bolt_class_name)'
+  def initialize(base_class_path, real_bolt_class_name)
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  end
+
+  java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
+  def prepare(conf, context, collector, id)
+    @real_bolt.prepare(conf, context, collector, id)
+  end
+
+  java_signature 'void execute(Tuple)'
+  def execute(tuple)
+    @real_bolt.execute(tuple)
+  end
+
+  java_signature 'void finishBatch()'
+  def finishBatch
+    @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_bolt.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
+end

data/lib/red_storm/proxy/batch_committer_bolt.rb

@@ -0,0 +1,52 @@
+require 'java'
+
+java_import 'backtype.storm.coordination.BatchOutputCollector'
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.coordination.IBatchBolt'
+java_import 'backtype.storm.transactional.ICommitter'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'backtype.storm.tuple.Tuple'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+class BatchCommitterBolt
+  java_implements 'ICommitter, IBatchBolt'
+
+  java_signature 'IBatchCommitterBolt (String base_class_path, String real_bolt_class_name)'
+  def initialize(base_class_path, real_bolt_class_name)
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_bolt = Object.module_eval(real_bolt_class_name).new
+  end
+
+  java_signature 'void prepare(Map, TopologyContext, BatchOutputCollector, Object)'
+  def prepare(conf, context, collector, id)
+    @real_bolt.prepare(conf, context, collector, id)
+  end
+
+  java_signature 'void execute(Tuple)'
+  def execute(tuple)
+    @real_bolt.execute(tuple)
+  end
+
+  java_signature 'void finishBatch()'
+  def finishBatch
+    @real_bolt.finish_batch if @real_bolt.respond_to?(:finish_batch)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_bolt.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_bolt.get_component_configuration
+  end
+end

data/lib/red_storm/proxy/batch_spout.rb

@@ -1,31 +1,19 @@
 require 'java'
 
-java_import 'storm.trident.operation.TridentCollector'
 java_import 'backtype.storm.task.TopologyContext'
+java_import 'storm.trident.operation.TridentCollector'
 java_import 'storm.trident.spout.IBatchSpout'
-java_import 'backtype.storm.topology.OutputFieldsDeclarer'
-java_import 'backtype.storm.tuple.Tuple'
 java_import 'backtype.storm.tuple.Fields'
-java_import 'backtype.storm.tuple.Values'
 java_import 'java.util.Map'
+
 module Backtype
   java_import 'backtype.storm.Config'
 end
 
 java_package 'redstorm.proxy'
 
-# the BatchSpout class is a proxy to the real batch spout to avoid having to deal with all the
+# the Spout class is a proxy to the real spout to avoid having to deal with all the
 # Java artifacts when creating a spout.
-#
-# The real batch spout class implementation must define these methods:
-# - open(conf, context, collector)
-# - emitBatch
-# - getOutputFields
-# - ack(batch_id)
-#
-# and optionnaly:
-# - close
-#
 
 class BatchSpout
   java_implements IBatchSpout
@@ -40,7 +28,12 @@ class BatchSpout
 
   java_signature 'void open(Map, TopologyContext)'
   def open(conf, context)
-    @real_spout.open(conf, context)
+    @real_spout.open(conf, context) if @real_spout.respond_to?(:open)
+  end
+
+  java_signature 'void emitBatch(long, TridentCollector)'
+  def emitBatch(batch_id, collector)
+  	@real_spout.emit_batch(batch_id, collector)
   end
 
   java_signature 'void close()'
@@ -48,19 +41,14 @@ class BatchSpout
     @real_spout.close if @real_spout.respond_to?(:close)
   end
 
-  java_signature 'void emitBatch(long, TridentCollector)'
-  def emitBatch(batch_id, collector)
-    @real_spout.emit_batch(batch_id, collector)
-  end
-
   java_signature 'void ack(long)'
   def ack(batch_id)
-    @real_spout.ack(batch_id)
+    @real_spout.ack(batch_id) if @real_spout.respond_to?(:ack)
   end
 
   java_signature 'Fields getOutputFields()'
-  def getOutputFields
-    @real_spout.get_output_fields()
+  def getOutputFields()
+    @real_spout.get_output_fields
   end
 
   java_signature 'Map<String, Object> getComponentConfiguration()'

data/lib/red_storm/proxy/proxy_function.rb

@@ -1,16 +1,10 @@
 require 'java'
 
-
 java_import 'storm.trident.tuple.TridentTuple'
-
 java_import 'storm.trident.operation.TridentCollector'
-
-java_import 'java.util.Map'
-
 java_import 'storm.trident.operation.TridentOperationContext'
-
 java_import 'storm.trident.operation.Function'
-
+java_import 'java.util.Map'
 
 module Backtype
   java_import 'backtype.storm.Config'
@@ -43,6 +37,4 @@ class ProxyFunction
   def prepare(_map, _trident_operation_context)
     @real.prepare(_map, _trident_operation_context)
   end
-
-
 end

data/lib/red_storm/proxy/transactional_committer_spout.rb

@@ -0,0 +1,47 @@
+require 'java'
+
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.transactional.ITransactionalSpout'
+java_import 'backtype.storm.transactional.ICommitterTransactionalSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+
+class TransactionalCommitterSpout
+  java_implements 'ICommitterTransactionalSpout'
+
+  java_signature 'ICommitterTransactionalSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'ICommitterTransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
+  def getEmitter(conf, context)
+    @real_spout.get_emitter(conf, context)
+  end
+
+  java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
+  def getCoordinator(conf, context)
+    @real_spout.get_coordinator(conf, context)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_spout.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/proxy/transactional_spout.rb

@@ -0,0 +1,46 @@
+require 'java'
+
+java_import 'backtype.storm.task.TopologyContext'
+java_import 'backtype.storm.transactional.ITransactionalSpout'
+java_import 'backtype.storm.topology.OutputFieldsDeclarer'
+java_import 'java.util.Map'
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
+java_package 'redstorm.proxy'
+
+
+class TransactionalSpout
+  java_implements 'ITransactionalSpout'
+
+  java_signature 'ITransactionalSpout (String base_class_path, String real_spout_class_name)'
+  def initialize(base_class_path, real_spout_class_name)
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  rescue NameError
+    require base_class_path
+    @real_spout = Object.module_eval(real_spout_class_name).new
+  end
+
+  java_signature 'ITransactionalSpout.Emitter getEmitter(Map, TopologyContext)'
+  def getEmitter(conf, context)
+    @real_spout.get_emitter(conf, context)
+  end
+
+  java_signature 'ITransactionalSpout.Coordinator getCoordinator(Map, TopologyContext)'
+  def getCoordinator(conf, context)
+    @real_spout.get_coordinator(conf, context)
+  end
+
+  java_signature 'void declareOutputFields(OutputFieldsDeclarer)'
+  def declareOutputFields(declarer)
+    @real_spout.declare_output_fields(declarer)
+  end
+
+  java_signature 'Map<String, Object> getComponentConfiguration()'
+  def getComponentConfiguration
+    @real_spout.get_component_configuration
+  end
+
+end

data/lib/red_storm/simple_drpc_topology.rb

@@ -13,11 +13,10 @@ module RedStorm
     end
 
     def grouping(grouping)
-      @grouping = @grouping
+      @grouping = grouping
     end
 
     def define_grouping(declarer)
-
       case @grouping
       when :fields
         declarer.fieldsGrouping(Fields.new(*([params].flatten.map(&:to_s))))
@@ -73,6 +72,7 @@ module RedStorm
     end
 
     def self.input_bolt(bolt_class, *args, &bolt_block)
+      set_topology_class!
       options = args.last.is_a?(Hash) ? args.pop : {}
       contructor_args = !args.empty? ? args.pop : []
       bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)