redstorm 0.6.5 → 0.6.6.beta1

data/lib/red_storm/environment.rb

@@ -1,3 +1,6 @@
+require 'java'
+java_import 'java.lang.System'
+
 module RedStorm
 
   LAUNCH_PATH = File.expand_path(File.dirname(__FILE__))
@@ -12,7 +15,7 @@ module RedStorm
     REDSTORM_HOME = File.expand_path(LAUNCH_PATH + '/../..')
     TARGET_PATH = "#{BASE_PATH}/target"
   end
-  
+
   unless defined?(SPECS_CONTEXT)
     GEM_PATH = "#{TARGET_PATH}/gems/"
     ENV["GEM_PATH"] = GEM_PATH
@@ -30,12 +33,9 @@ module RedStorm
     version_map[ruby_version.to_s] || version_map[RedStorm.current_ruby_mode]
   end
 
-  module_function :current_ruby_mode, :jruby_mode_token
+  def java_runtime_version
+    System.properties["java.runtime.version"].to_s =~ /^(\d+\.\d+).[^\s]+$/ ? $1 : "1.7"
+  end
 
-  # puts("*** LAUNCH_PATH=#{LAUNCH_PATH}")
-  # puts("*** JAR_CONTEXT=#{JAR_CONTEXT}")
-  # puts("*** BASE_PATH=#{BASE_PATH}")
-  # puts("*** REDSTORM_HOME=#{REDSTORM_HOME}")
-  # puts("*** TARGET_PATH=#{TARGET_PATH}")
-  # puts("*** GEM_PATH=#{GEM_PATH}") if defined?(GEM_PATH)
+  module_function :current_ruby_mode, :jruby_mode_token, :java_runtime_version
 end

data/lib/red_storm/topology_launcher.rb

@@ -1,7 +1,7 @@
 require 'java'
 
 # This hack get rif of the "Use RbConfig instead of obsolete and deprecated Config"
-# deprecation warning that is triggered by "java_import 'backtype.storm.Config'". 
+# deprecation warning that is triggered by "java_import 'backtype.storm.Config'".
 Object.send :remove_const, :Config
 Config = RbConfig
 
@@ -59,7 +59,7 @@ class TopologyLauncher
 
     topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
     puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
-    RedStorm::Configuration.topology_class.new.start(class_path, env)
+    RedStorm::Configuration.topology_class.new.start(env)
   end
 
   private

data/lib/red_storm/version.rb

@@ -1,3 +1,3 @@
 module RedStorm
-  VERSION = '0.6.5'
+  VERSION = '0.6.6.beta1'
 end

data/lib/tasks/red_storm.rake

@@ -7,7 +7,7 @@ rescue
 end
 
 require 'jruby/jrubyc'
-require 'red_storm'
+require 'red_storm/environment'
 require 'red_storm/application'
 
 INSTALL_IVY_VERSION = "2.3.0"
@@ -15,7 +15,7 @@ INSTALL_IVY_VERSION = "2.3.0"
 task :launch, :env, :ruby_mode, :class_file do |t, args|
   # use ruby mode parameter or default to current interpreter version
   version_token = RedStorm.jruby_mode_token(args[:ruby_mode])
-  
+
   command = case args[:env]
   when "local"
     RedStorm::Application.local_storm_command(args[:class_file], args[:ruby_mode])
@@ -43,22 +43,24 @@ end
 
 task :setup do
   puts("\n--> Setting up target directories")
-  ant.mkdir 'dir' => TARGET_DIR 
-  ant.mkdir 'dir' => TARGET_CLASSES_DIR 
+  ant.mkdir 'dir' => TARGET_DIR
+  ant.mkdir 'dir' => TARGET_CLASSES_DIR
   ant.mkdir 'dir' => TARGET_DEPENDENCY_DIR
   ant.mkdir 'dir' => TARGET_SRC_DIR
   ant.mkdir 'dir' => TARGET_GEM_DIR
   ant.mkdir 'dir' => TARGET_SPECS_DIR
-  ant.path 'id' => 'classpath' do  
-    fileset 'dir' => TARGET_DEPENDENCY_DIR  
-    fileset 'dir' => TARGET_CLASSES_DIR  
-  end  
+  ant.path 'id' => 'classpath' do
+    fileset 'dir' => TARGET_DEPENDENCY_DIR
+    fileset 'dir' => TARGET_CLASSES_DIR
+  end
 end
 
-task :install => [:deps, :build] do
+desc "install dependencies and compile proxy classes"
+task :install, [:jvm_version] => [:deps, :build] do |t, args|
   puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}")
 end
 
+desc "locally install examples"
 task :examples do
   if File.identical?(SRC_EXAMPLES, DST_EXAMPLES)
     STDERR.puts("error: cannot copy examples into itself")
@@ -78,23 +80,27 @@ task :copy_red_storm do
   FileUtils.cp_r(REDSTORM_LIB_DIR, TARGET_DIR)
 end
 
-task :build => [:setup, :copy_red_storm] do
+desc "compile JRuby and Java proxy classes"
+task :build, [:jvm_version] => [:setup, :copy_red_storm] do |t, args|
+  jvm_version = args[:jvm_version].to_s =~ /--(1.\d)/ ? $1 : RedStorm.java_runtime_version
+
   # compile the JRuby proxy classes to Java
   build_jruby("#{REDSTORM_LIB_DIR}/red_storm/proxy")
 
   # compile the generated Java proxy classes
-  build_java_dir("#{TARGET_SRC_DIR}")
+  build_java_dir("#{TARGET_SRC_DIR}", jvm_version)
 
   # generate the JRuby topology launcher
   build_jruby("#{REDSTORM_LIB_DIR}/red_storm/topology_launcher.rb")
 
   # compile the JRuby proxy classes
-  build_java_dir("#{REDSTORM_JAVA_SRC_DIR}")
+  build_java_dir("#{REDSTORM_JAVA_SRC_DIR}", jvm_version)
 
   # compile the JRuby proxy classes
-  build_java_dir("#{TARGET_SRC_DIR}")
+  build_java_dir("#{TARGET_SRC_DIR}", jvm_version)
 end
 
+desc "package topology gems into #{TARGET_GEM_DIR}"
 task :bundle, [:groups] => :setup do |t, args|
   require 'bundler'
   defaulted_args = {:groups => 'default'}.merge(args.to_hash.delete_if{|k, v| v.to_s.empty?})
@@ -143,18 +149,29 @@ namespace :ivy do
   end
 end
 
-task :deps => "ivy:install" do
-  puts("\n--> Installing dependencies")
-
+task :ivy_config do
   ant.configure 'file' => File.exists?(CUSTOM_IVY_SETTINGS) ? CUSTOM_IVY_SETTINGS : DEFAULT_IVY_SETTINGS
+end
 
-  ant.resolve 'file' => File.exists?(CUSTOM_IVY_STORM_DEPENDENCIES) ? CUSTOM_IVY_STORM_DEPENDENCIES : DEFAULT_IVY_STORM_DEPENDENCIES 
-  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/storm/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"  
+task :storm_deps => ["ivy:install", :ivy_config] do
+  puts("\n--> Installing Storm dependencies")
 
-  ant.resolve 'file' => File.exists?(CUSTOM_IVY_TOPOLOGY_DEPENDENCIES) ? CUSTOM_IVY_TOPOLOGY_DEPENDENCIES : DEFAULT_IVY_TOPOLOGY_DEPENDENCIES 
-  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/topology/[conf]/[artifact]-[revision].[ext]", 'sync' => "true"
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_STORM_DEPENDENCIES) ? CUSTOM_IVY_STORM_DEPENDENCIES : DEFAULT_IVY_STORM_DEPENDENCIES
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/storm/[conf]/[artifact](-[classifier])-[revision].[ext]", 'sync' => "true"
 end
 
+task :topology_deps => ["ivy:install", :ivy_config] do
+  puts("\n--> Installing topology dependencies")
+
+  ant.resolve 'file' => File.exists?(CUSTOM_IVY_TOPOLOGY_DEPENDENCIES) ? CUSTOM_IVY_TOPOLOGY_DEPENDENCIES : DEFAULT_IVY_TOPOLOGY_DEPENDENCIES
+  ant.retrieve 'pattern' => "#{TARGET_DEPENDENCY_DIR}/topology/[conf]/[artifact](-[classifier])-[revision].[ext]", 'sync' => "true"
+end
+
+desc "install storm and topology dependencies in #{TARGET_DEPENDENCY_DIR}"
+task :deps => ["ivy:install", :ivy_config, :storm_deps, :topology_deps] do
+end
+
+desc "generate #{TARGET_CLUSTER_JAR}"
 task :jar, [:include_dir] => [:clean_jar] do |t, args|
   puts("\n--> Generating JAR file #{TARGET_CLUSTER_JAR}")
 
@@ -199,22 +216,23 @@ task :jar, [:include_dir] => [:clean_jar] do |t, args|
   puts("\nRedStorm generated JAR file #{TARGET_CLUSTER_JAR}")
 end
 
-def build_java_dir(source_folder)
-  puts("\n--> Compiling Java")
+def build_java_dir(source_folder, jvm_version)
+  puts("\n--> Compiling Java for JVM #{jvm_version}")
   ant.javac(
     'srcdir' => source_folder,
     'destdir' => TARGET_CLASSES_DIR,
-    'classpathref' => 'classpath', 
-    'source' => "1.7",
-    'target' => "1.7",
+    'classpathref' => 'classpath',
+    'source' => jvm_version,
+    'target' => jvm_version,
     'debug' => "yes",
     'includeantruntime' => "no",
     'verbose' => false,
     'listfiles' => true
   ) do
+    # compilerarg :value => "-Xlint:deprecation"
     # compilerarg :value => "-Xlint:unchecked"
-  end 
-end  
+  end
+end
 
 def build_jruby(source_path)
   puts("\n--> Compiling JRuby")

data/redstorm.gemspec

@@ -3,7 +3,7 @@ $:.unshift libdir unless $:.include?(libdir)
 
 require 'red_storm/version'
 
-Gem::Specification.new do |s|  
+Gem::Specification.new do |s|
   s.name        = 'redstorm'
   s.version     = RedStorm::VERSION
   s.authors     = ['Colin Surprenant']
@@ -11,14 +11,14 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/colinsurprenant/redstorm'
   s.summary     = 'JRuby on Storm'
   s.description = 'JRuby integration & DSL for the Storm distributed realtime computation system'
- 
+
   s.rubyforge_project = 'redstorm'
-  
+
   s.files         = Dir.glob("{lib/**/*}") + Dir.glob("{ivy/*.xml}") + Dir.glob("{examples/**/*}") + Dir.glob("{src/**/*.java}") + Dir.glob("{bin/**/*}") + %w(redstorm.gemspec Rakefile README.md CHANGELOG.md LICENSE.md)
   s.require_paths = ['lib']
   s.bindir        = 'bin'
   s.executables   = ['redstorm']
 
-  s.add_development_dependency 'rspec', '~> 2.11.0'
+  s.add_development_dependency 'rspec', '~> 2.13'
   s.add_runtime_dependency 'rake'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: redstorm
 version: !ruby/object:Gem::Version
-  version: 0.6.5
-  prerelease:
+  version: 0.6.6.beta1
+  prerelease: 6
 platform: ruby
 authors:
 - Colin Surprenant
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-05-15 00:00:00.000000000 Z
+date: 2013-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.11.0
+        version: '2.13'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.11.0
+        version: '2.13'
     none: false
   prerelease: false
   type: :development
@@ -31,17 +31,15 @@ dependencies:
   name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: !binary |-
-          MA==
+        version: '0'
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
-        version: !binary |-
-          MA==
+        version: '0'
     none: false
   prerelease: false
   type: :runtime
@@ -59,12 +57,13 @@ files:
 - lib/red_storm/configurator.rb
 - lib/red_storm/environment.rb
 - lib/red_storm/loggable.rb
-- lib/red_storm/simple_bolt.rb
-- lib/red_storm/simple_drpc_topology.rb
-- lib/red_storm/simple_spout.rb
-- lib/red_storm/simple_topology.rb
 - lib/red_storm/topology_launcher.rb
 - lib/red_storm/version.rb
+- lib/red_storm/dsl/bolt.rb
+- lib/red_storm/dsl/drpc_topology.rb
+- lib/red_storm/dsl/spout.rb
+- lib/red_storm/dsl/topology.rb
+- lib/red_storm/dsl/tuple.rb
 - lib/red_storm/proxy/batch_bolt.rb
 - lib/red_storm/proxy/batch_committer_bolt.rb
 - lib/red_storm/proxy/batch_spout.rb
@@ -77,6 +76,17 @@ files:
 - ivy/settings.xml
 - ivy/storm_dependencies.xml
 - ivy/topology_dependencies.xml
+- examples/dsl/exclamation_bolt.rb
+- examples/dsl/exclamation_topology.rb
+- examples/dsl/exclamation_topology2.rb
+- examples/dsl/hello_world_topology.rb
+- examples/dsl/kafka_topology.rb
+- examples/dsl/random_sentence_spout.rb
+- examples/dsl/redis_word_count_topology.rb
+- examples/dsl/ruby_version_topology.rb
+- examples/dsl/split_sentence_bolt.rb
+- examples/dsl/word_count_bolt.rb
+- examples/dsl/word_count_topology.rb
 - examples/native/cluster_word_count_topology.rb
 - examples/native/exclamation_bolt.rb
 - examples/native/local_exclamation_topology.rb
@@ -89,17 +99,6 @@ files:
 - examples/shell/shell_topology.rb
 - examples/shell/resources/splitsentence.py
 - examples/shell/resources/storm.py
-- examples/simple/exclamation_bolt.rb
-- examples/simple/exclamation_topology.rb
-- examples/simple/exclamation_topology2.rb
-- examples/simple/hello_world_topology.rb
-- examples/simple/kafka_topology.rb
-- examples/simple/random_sentence_spout.rb
-- examples/simple/redis_word_count_topology.rb
-- examples/simple/ruby_version_topology.rb
-- examples/simple/split_sentence_bolt.rb
-- examples/simple/word_count_bolt.rb
-- examples/simple/word_count_topology.rb
 - src/main/redstorm/storm/jruby/JRubyBatchBolt.java
 - src/main/redstorm/storm/jruby/JRubyBatchCommitterBolt.java
 - src/main/redstorm/storm/jruby/JRubyBatchSpout.java
@@ -126,17 +125,15 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
-      version: !binary |-
-        MA==
+      version: '0'
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>'
     - !ruby/object:Gem::Version
-      version: !binary |-
-        MA==
+      version: 1.3.1
   none: false
 requirements: []
 rubyforge_project: redstorm

data/examples/simple/exclamation_bolt.rb

@@ -1,10 +0,0 @@
-require 'red_storm'
-
-module RedStorm
-  module Examples
-    class ExclamationBolt < RedStorm::SimpleBolt
-      output_fields :word
-      on_receive(:ack => true, :anchor => true) {|tuple| tuple.getString(0) + "!!!"}
-    end
-  end
-end

data/lib/red_storm/simple_bolt.rb

@@ -1,135 +0,0 @@
-require 'java'
-require 'red_storm/configurator'
-
-module RedStorm
-
-  class SimpleBolt
-    attr_reader :collector, :context, :config
-
-    # DSL class methods
-
-    def self.log
-      @log ||= Java::OrgApacheLog4j::Logger.getLogger(self.name)
-    end
-
-    def self.output_fields(*fields)
-      @fields = fields.map(&:to_s)
-    end
-
-    def self.configure(&configure_block)
-      @configure_block = block_given? ? configure_block : lambda {}
-    end
-
-    def self.on_receive(*args, &on_receive_block)
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      method_name = args.first
-
-      self.receive_options.merge!(options)
-      @on_receive_block = block_given? ? on_receive_block : lambda {|tuple| self.send(method_name || :on_receive, tuple)}
-    end
-
-    def self.on_init(method_name = nil, &on_init_block)
-      @on_init_block = block_given? ? on_init_block : lambda {self.send(method_name || :on_init)}
-    end
-
-    def self.on_close(method_name = nil, &close_block)
-      @close_block = block_given? ? close_block : lambda {self.send(method_name || :on_close)}
-    end
-
-    # DSL instance methods
-
-    def log
-      self.class.log
-    end
-
-    def unanchored_emit(*values)
-      @collector.emit(Values.new(*values)) 
-    end
-
-    def anchored_emit(tuple, *values)
-      @collector.emit(tuple, Values.new(*values)) 
-    end
-
-    def ack(tuple)
-      @collector.ack(tuple)
-    end
-
-    def fail(tuple)
-      @collector.fail(tuple)
-    end
-
-    # Bolt proxy interface
-
-    def execute(tuple)
-      output = instance_exec(tuple, &self.class.on_receive_block)
-      if output && self.class.emit?
-        values_list = !output.is_a?(Array) ? [[output]] : !output.first.is_a?(Array) ? [output] : output
-        values_list.each{|values| self.class.anchor? ? anchored_emit(tuple, *values) : unanchored_emit(*values)}
-        @collector.ack(tuple) if self.class.ack?
-      end
-    end
-
-    def prepare(config, context, collector)
-      @collector = collector
-      @context = context
-      @config = config
-      instance_exec(&self.class.on_init_block)
-    end
-
-    def cleanup
-      instance_exec(&self.class.close_block)
-    end
-
-    def declare_output_fields(declarer)
-      declarer.declare(Fields.new(self.class.fields))
-    end
-
-    def get_component_configuration
-      configurator = Configurator.new
-      configurator.instance_exec(&self.class.configure_block)
-      configurator.config
-    end
-
-    private
-
-    # default noop optional dsl callbacks
-    def on_init; end
-    def on_close; end
-
-    def self.fields
-      @fields ||= []
-    end
-
-    def self.configure_block
-      @configure_block ||= lambda {}
-    end
-
-    def self.on_receive_block
-      @on_receive_block ||= lambda {|tuple| self.send(:on_receive, tuple)}
-    end
-
-    def self.on_init_block
-      @on_init_block ||= lambda {self.send(:on_init)}
-    end
-
-    def self.close_block
-      @close_block ||= lambda {self.send(:on_close)}
-    end
-
-    def self.receive_options
-      @receive_options ||= {:emit => true, :ack => false, :anchor => false}
-    end
-
-    def self.emit?
-      !!self.receive_options[:emit]
-    end
-
-    def self.ack?
-      !!self.receive_options[:ack]
-    end
-
-    def self.anchor?
-      !!self.receive_options[:anchor]
-    end
-  end
-end