redstorm 0.4.0 → 0.5.0

data/lib/red_storm/simple_topology.rb

@@ -1,25 +1,37 @@
 require 'red_storm/configuration'
+require 'red_storm/configurator'
 
 module RedStorm
 
+  class TopologyDefinitionError < StandardError; end
+
   class SimpleTopology
     attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
 
     DEFAULT_SPOUT_PARALLELISM = 1
     DEFAULT_BOLT_PARALLELISM = 1
 
-    class ComponentDefinition
+    class ComponentDefinition < Configurator
       attr_reader :clazz, :parallelism
       attr_accessor :id # ids are forced to string
 
       def initialize(component_class, id, parallelism)
+        super()
         @clazz = component_class
         @id = id.to_s
         @parallelism = parallelism
       end
+
+      def is_java?
+        @clazz.name.split('::').first.downcase == 'java'
+      end
     end
 
-    class SpoutDefinition < ComponentDefinition; end
+    class SpoutDefinition < ComponentDefinition
+      def new_instance(base_class_path)
+        is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
+      end
+    end
           
     class BoltDefinition < ComponentDefinition
       attr_accessor :sources
@@ -55,36 +67,27 @@ module RedStorm
           end
         end
       end
-    end
-
-    class Configurator
-      attr_reader :config
-
-      def initialize
-        @config = Config.new
-      end
 
-      def method_missing(sym, *args)
-        config_method = "set#{self.class.camel_case(sym)}"
-        @config.send(config_method, *args)
+      def new_instance(base_class_path)
+        is_java? ? @clazz.new : JRubyBolt.new(base_class_path, @clazz.name)
       end
+    end
 
-      private
-
-      def self.camel_case(s)
-        s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
-      end
+    def self.log
+      @log ||= org.apache.log4j.Logger.getLogger(self.name)
     end
 
-    def self.spout(spout_class, options = {})
+    def self.spout(spout_class, options = {}, &spout_block)
       spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
       spout = SpoutDefinition.new(spout_class, spout_options[:id], spout_options[:parallelism])
+      spout.instance_exec(&spout_block) if block_given?
       self.components << spout
     end
 
     def self.bolt(bolt_class, options = {}, &bolt_block)
       bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
       bolt = BoltDefinition.new(bolt_class, bolt_options[:id], bolt_options[:parallelism])
+      raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
       bolt.instance_exec(&bolt_block)
       self.components << bolt
     end
@@ -106,12 +109,12 @@ module RedStorm
 
       builder = TopologyBuilder.new
       self.class.spouts.each do |spout|
-        is_java = spout.clazz.name.split('::').first == 'Java'
-        builder.setSpout(spout.id, is_java ? spout.clazz.new : JRubySpout.new(base_class_path, spout.clazz.name), spout.parallelism)
+        declarer = builder.setSpout(spout.id, spout.new_instance(base_class_path), spout.parallelism)
+        declarer.addConfigurations(spout.config)
       end
       self.class.bolts.each do |bolt|
-        is_java = bolt.clazz.name.split('::').first == 'Java'
-        declarer = builder.setBolt(bolt.id, is_java ? bolt.clazz.new : JRubyBolt.new(base_class_path, bolt.clazz.name), bolt.parallelism)
+        declarer = builder.setBolt(bolt.id, bolt.new_instance(base_class_path), bolt.parallelism)
+        declarer.addConfigurations(bolt.config)
         bolt.define_grouping(declarer)
       end
 
@@ -173,4 +176,4 @@ module RedStorm
       camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
     end
   end
-end
+end

data/lib/red_storm/topology_launcher.rb

@@ -10,7 +10,11 @@ rescue LoadError
   require 'red_storm'
 end
 
-java_import 'backtype.storm.Config'
+# see https://github.com/colinsurprenant/redstorm/issues/7
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
 java_import 'backtype.storm.LocalCluster'
 java_import 'backtype.storm.StormSubmitter'
 java_import 'backtype.storm.topology.TopologyBuilder'
@@ -36,7 +40,7 @@ class TopologyLauncher
     env = args[0].to_sym
     class_path = args[1]
 
-    require class_path
+    require "./#{class_path}" # ./ for 1.9 compatibility
 
     topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
     puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/lib/tasks/red_storm.rake

@@ -9,6 +9,10 @@ rescue LoadError
   require 'red_storm'
 end
 
+INSTALL_STORM_VERSION = "0.7.1"
+INSTALL_JRUBY_VERSION = "1.6.7"
+DEFAULT_GEMFILE = "Gemfile"
+
 CWD = Dir.pwd
 TARGET_DIR = "#{CWD}/target"
 TARGET_SRC_DIR = "#{TARGET_DIR}/src"
@@ -26,8 +30,9 @@ SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples"
 DST_EXAMPLES = "#{CWD}/examples"
 
 task :launch, :env, :class_file do |t, args|
+  version_token = RedStorm::RUNTIME['RUBY_VERSION'] == "--1.9" ? "RUBY1_9" : "RUBY1_8"
   gem_home = ENV["GEM_HOME"].to_s.empty? ? " -Djruby.gem.home=`gem env home`" : ""
-  command = "java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}"
+  command = "java -Djruby.compat.version=#{version_token} -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}"
   puts("launching #{command}")
   system(command)
 end
@@ -40,53 +45,46 @@ task :clean_jar do
   ant.delete :file => TARGET_CLUSTER_JAR
 end
 
-task :setup do  
+task :setup do
+  puts("\n--> Setting up target directories")
   ant.mkdir :dir => TARGET_DIR 
   ant.mkdir :dir => TARGET_CLASSES_DIR 
   ant.mkdir :dir => TARGET_SRC_DIR
   ant.mkdir :dir => TARGET_GEMS_DIR
+  ant.mkdir :dir => "#{TARGET_GEMS_DIR}/gems"
+  ant.mkdir :dir => "#{TARGET_GEMS_DIR}/bundler"
   ant.path :id => 'classpath' do  
     fileset :dir => TARGET_DEPENDENCY_DIR  
     fileset :dir => TARGET_CLASSES_DIR  
   end  
 end
 
-task :install => [:deps, :build] do
+task :install => [:deps, :build, :gems] do
   puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}")
 end
 
 task :unpack do
-  system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
+  system("rmvn dependency:unpack \
+          -f #{RedStorm::REDSTORM_HOME}/pom.xml \
+          -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR} \
+          -DmarkersDirectory=#{TARGET_MARKERS_DIR} \
+          -Dstorm-storm.version=#{INSTALL_STORM_VERSION} \
+          -Dorg.jruby-jruby-complete.version=#{INSTALL_JRUBY_VERSION}")
 end
 
-task :devjar => [:unpack, :clean_jar] do
+task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
   ant.jar :destfile => TARGET_CLUSTER_JAR do
     fileset :dir => TARGET_CLASSES_DIR
     fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
-    fileset :dir => TARGET_GEMS_DIR
-    fileset :dir => RedStorm::REDSTORM_HOME do
-      include :name => "examples/**/*"
+    fileset :dir => TARGET_GEMS_DIR do
+      # remove bundler config dir to avoid setting BUNDLE_PATH
+      exclude :name => "bundler/.bundle/**"
     end
     fileset :dir => JRUBY_SRC_DIR do
       exclude :name => "tasks/**"
     end
-    manifest do
-      attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
-    end
-  end
-  puts("\nRedStorm generated dev jar file #{TARGET_CLUSTER_JAR}")
-end
-
-task :jar, [:dir] => [:unpack, :clean_jar] do |t, args|
-  ant.jar :destfile => TARGET_CLUSTER_JAR do
-    fileset :dir => TARGET_CLASSES_DIR
-    fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
-    fileset :dir => TARGET_GEMS_DIR
-    fileset :dir => JRUBY_SRC_DIR do
-      exclude :name => "tasks/**"
-    end
     fileset :dir => CWD do
-      include :name => "#{args[:dir]}/**/*"
+      args[:include_dir].split(":").each{|dir| include :name => "#{dir}/**/*"}
     end
     manifest do
       attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
@@ -111,8 +109,14 @@ task :examples do
   puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}")
 end
 
-task :deps do
-  system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
+task :deps => :setup do
+  puts("\n--> Installing dependencies")
+  system("rmvn dependency:copy-dependencies \
+          -f #{RedStorm::REDSTORM_HOME}/pom.xml \
+          -DoutputDirectory=#{TARGET_DEPENDENCY_DIR} \
+          -DmarkersDirectory=#{TARGET_MARKERS_DIR} \
+          -Dstorm-storm.version=#{INSTALL_STORM_VERSION} \
+          -Dorg.jruby-jruby-complete.version=#{INSTALL_JRUBY_VERSION}")
 end
 
 task :build => :setup do
@@ -130,7 +134,31 @@ task :build => :setup do
 
   # compile the JRuby proxy classes
   build_java_dir("#{TARGET_SRC_DIR}")
-end 
+end
+
+task :gems, [:bundler_options] => :setup do |t, args|
+  bundler_options = args[:bundler_options].to_s.split(":").join(" ")
+
+  # basically copy original Gemfile to target/gems/bundler and install into this dir
+  gemfile = bundler_options =~ /--gemfile\s+([^\s]+)/ ? $1 : DEFAULT_GEMFILE
+  if bundler_options =~ /--gemfile\s+[^\s]+/
+    bundler_options.gsub!(/--gemfile\s+[^\s]+/, "--gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile")
+  else
+    bundler_options = bundler_options + " --gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile"
+  end
+
+  puts("\n--> Installing gems in #{TARGET_GEMS_DIR}/gems")
+  system("gem install bundler --install-dir #{TARGET_GEMS_DIR}/gems --no-ri --no-rdoc --quiet --no-verbose")
+  system("gem install rake --install-dir #{TARGET_GEMS_DIR}/gems --no-ri --no-rdoc --quiet --no-verbose")
+
+  if File.exist?(gemfile)
+    puts("\n--> Bundling gems in #{TARGET_GEMS_DIR}/bundler using #{gemfile}")
+    system("cp #{gemfile} #{TARGET_GEMS_DIR}/bundler/Gemfile")
+    system("jruby #{RedStorm::RUNTIME['RUBY_VERSION']} -S bundle install #{bundler_options} --path #{TARGET_GEMS_DIR}/bundler/")
+  elsif gemfile != DEFAULT_GEMFILE
+    puts("WARNING: #{gemfile} not found, cannot bundle gems")
+  end
+end
 
 def build_java_dir(source_folder)
   puts("\n--> Compiling Java")
@@ -144,10 +172,12 @@ def build_java_dir(source_folder)
     :includeantruntime => "no",
     :verbose => false,
     :listfiles => true
-  )
+  ) do
+    # compilerarg :value => "-Xlint:unchecked"
+  end 
 end  
 
 def build_jruby(source_path)
   puts("\n--> Compiling JRuby")
-  system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.6.2.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
+  system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-#{INSTALL_STORM_VERSION}.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
 end

data/pom.xml

@@ -2,10 +2,11 @@
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
+
   <groupId>redstorm</groupId>
   <artifactId>redstorm</artifactId>
-  <version>0.4.0</version>
-  <name>RedStorm JRuby on Storm</name>
+  <version>0.5.0</version>
+  <name>RedStorm</name>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -27,13 +28,13 @@
     <dependency>
       <groupId>storm</groupId>
       <artifactId>storm</artifactId>
-      <version>0.6.2</version>
+      <version>${storm-storm.version}</version>
     </dependency>
   
     <dependency>
       <groupId>org.jruby</groupId>
       <artifactId>jruby-complete</artifactId>
-      <version>1.6.6</version>
+      <version>${org.jruby-jruby-complete.version}</version>
     </dependency>
   </dependencies>
 
@@ -48,7 +49,7 @@
             <artifactItem>
               <groupId>org.jruby</groupId>
               <artifactId>jruby-complete</artifactId>
-              <version>1.6.6</version>
+              <version>${org.jruby-jruby-complete.version}</version>
               <type>jar</type>
               <overWrite>false</overWrite>
             </artifactItem>

data/src/main/redstorm/storm/jruby/JRubyBolt.java

@@ -58,6 +58,16 @@ public class JRubyBolt implements IRichBolt {
     bolt.declareOutputFields(declarer);
   }
 
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time, before serialisation.
+    // do not set the _proxyBolt instance variable here to avoid JRuby serialization
+    // issues. Just create tmp bolt instance to call declareOutputFields.
+    IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
+    return bolt.getComponentConfiguration();
+  }
+ 
+
   private static IRichBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
     try {
       redstorm.proxy.Bolt proxy = new redstorm.proxy.Bolt(baseClassPath, realBoltClassName);

data/src/main/redstorm/storm/jruby/JRubySpout.java

@@ -33,15 +33,6 @@ public class JRubySpout implements IRichSpout {
     _realSpoutClassName = realSpoutClassName;
   }
 
-  @Override
-  public boolean isDistributed() {
-    // isDistributed is executed in the topology creation time before serialisation.
-    // do not set the _proxySpout instance variable here to avoid JRuby serialization
-    // issues. Just create tmp spout instance to call isDistributed.
-    IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
-    return spout.isDistributed();
-  }
-
   @Override
   public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
     // create instance of the jruby proxy class here, after deserialization in the workers.
@@ -54,6 +45,16 @@ public class JRubySpout implements IRichSpout {
     _proxySpout.close();
   }
 
+  @Override
+  public void activate() {
+    _proxySpout.activate();
+  }
+
+  @Override
+  public void deactivate() {
+    _proxySpout.deactivate();
+  }
+
   @Override
   public void nextTuple() {
     _proxySpout.nextTuple();
@@ -78,6 +79,15 @@ public class JRubySpout implements IRichSpout {
     spout.declareOutputFields(declarer);
   }  
 
+  @Override
+  public Map<String, Object> getComponentConfiguration() {
+    // getComponentConfiguration is executed in the topology creation time before serialisation.
+    // do not set the _proxySpout instance variable here to avoid JRuby serialization
+    // issues. Just create tmp spout instance to call declareOutputFields.
+    IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
+    return spout.getComponentConfiguration();
+  }
+ 
   private static IRichSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
     try {
       redstorm.proxy.Spout proxy = new redstorm.proxy.Spout(baseClassPath, realSpoutClassName);

metadata

@@ -2,7 +2,7 @@
 name: redstorm
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
   - Colin Surprenant
@@ -10,17 +10,17 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-08 00:00:00 Z
+date: 2012-06-01 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
-    name: rubyforge
+    name: rspec
     prerelease: false
     requirement: &id001 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
+        - - ~>
           - !ruby/object:Gem::Version 
-            version: "0"
+            version: 2.8.0
     type: :development
     version_requirements: *id001
   - !ruby/object:Gem::Dependency 
@@ -29,9 +29,9 @@ dependencies:
     requirement: &id002 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ~>
+        - - ">="
           - !ruby/object:Gem::Version 
-            version: 0.9.2
+            version: "0"
     type: :runtime
     version_requirements: *id002
   - !ruby/object:Gem::Dependency 
@@ -58,6 +58,8 @@ files:
   - lib/red_storm.rb
   - lib/red_storm/application.rb
   - lib/red_storm/configuration.rb
+  - lib/red_storm/configurator.rb
+  - lib/red_storm/loggable.rb
   - lib/red_storm/simple_bolt.rb
   - lib/red_storm/simple_spout.rb
   - lib/red_storm/simple_topology.rb
@@ -78,8 +80,10 @@ files:
   - examples/simple/exclamation_bolt.rb
   - examples/simple/exclamation_topology.rb
   - examples/simple/exclamation_topology2.rb
+  - examples/simple/Gemfile
   - examples/simple/random_sentence_spout.rb
   - examples/simple/redis_word_count_topology.rb
+  - examples/simple/ruby_version_topology.rb
   - examples/simple/split_sentence_bolt.rb
   - examples/simple/word_count_bolt.rb
   - examples/simple/word_count_topology.rb
@@ -111,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.3.0
+        version: "0"
 requirements: []
 
 rubyforge_project: redstorm