redstorm 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,25 +1,37 @@
1
1
  require 'red_storm/configuration'
2
+ require 'red_storm/configurator'
2
3
 
3
4
  module RedStorm
4
5
 
6
+ class TopologyDefinitionError < StandardError; end
7
+
5
8
  class SimpleTopology
6
9
  attr_reader :cluster # LocalCluster reference usable in on_submit block, for example
7
10
 
8
11
  DEFAULT_SPOUT_PARALLELISM = 1
9
12
  DEFAULT_BOLT_PARALLELISM = 1
10
13
 
11
- class ComponentDefinition
14
+ class ComponentDefinition < Configurator
12
15
  attr_reader :clazz, :parallelism
13
16
  attr_accessor :id # ids are forced to string
14
17
 
15
18
  def initialize(component_class, id, parallelism)
19
+ super()
16
20
  @clazz = component_class
17
21
  @id = id.to_s
18
22
  @parallelism = parallelism
19
23
  end
24
+
25
+ def is_java?
26
+ @clazz.name.split('::').first.downcase == 'java'
27
+ end
20
28
  end
21
29
 
22
- class SpoutDefinition < ComponentDefinition; end
30
+ class SpoutDefinition < ComponentDefinition
31
+ def new_instance(base_class_path)
32
+ is_java? ? @clazz.new : JRubySpout.new(base_class_path, @clazz.name)
33
+ end
34
+ end
23
35
 
24
36
  class BoltDefinition < ComponentDefinition
25
37
  attr_accessor :sources
@@ -55,36 +67,27 @@ module RedStorm
55
67
  end
56
68
  end
57
69
  end
58
- end
59
-
60
- class Configurator
61
- attr_reader :config
62
-
63
- def initialize
64
- @config = Config.new
65
- end
66
70
 
67
- def method_missing(sym, *args)
68
- config_method = "set#{self.class.camel_case(sym)}"
69
- @config.send(config_method, *args)
71
+ def new_instance(base_class_path)
72
+ is_java? ? @clazz.new : JRubyBolt.new(base_class_path, @clazz.name)
70
73
  end
74
+ end
71
75
 
72
- private
73
-
74
- def self.camel_case(s)
75
- s.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
76
- end
76
+ def self.log
77
+ @log ||= org.apache.log4j.Logger.getLogger(self.name)
77
78
  end
78
79
 
79
- def self.spout(spout_class, options = {})
80
+ def self.spout(spout_class, options = {}, &spout_block)
80
81
  spout_options = {:id => self.underscore(spout_class), :parallelism => DEFAULT_SPOUT_PARALLELISM}.merge(options)
81
82
  spout = SpoutDefinition.new(spout_class, spout_options[:id], spout_options[:parallelism])
83
+ spout.instance_exec(&spout_block) if block_given?
82
84
  self.components << spout
83
85
  end
84
86
 
85
87
  def self.bolt(bolt_class, options = {}, &bolt_block)
86
88
  bolt_options = {:id => self.underscore(bolt_class), :parallelism => DEFAULT_BOLT_PARALLELISM}.merge(options)
87
89
  bolt = BoltDefinition.new(bolt_class, bolt_options[:id], bolt_options[:parallelism])
90
+ raise(TopologyDefinitionError, "#{bolt.clazz.name}, #{bolt.id}, bolt definition body required") unless block_given?
88
91
  bolt.instance_exec(&bolt_block)
89
92
  self.components << bolt
90
93
  end
@@ -106,12 +109,12 @@ module RedStorm
106
109
 
107
110
  builder = TopologyBuilder.new
108
111
  self.class.spouts.each do |spout|
109
- is_java = spout.clazz.name.split('::').first == 'Java'
110
- builder.setSpout(spout.id, is_java ? spout.clazz.new : JRubySpout.new(base_class_path, spout.clazz.name), spout.parallelism)
112
+ declarer = builder.setSpout(spout.id, spout.new_instance(base_class_path), spout.parallelism)
113
+ declarer.addConfigurations(spout.config)
111
114
  end
112
115
  self.class.bolts.each do |bolt|
113
- is_java = bolt.clazz.name.split('::').first == 'Java'
114
- declarer = builder.setBolt(bolt.id, is_java ? bolt.clazz.new : JRubyBolt.new(base_class_path, bolt.clazz.name), bolt.parallelism)
116
+ declarer = builder.setBolt(bolt.id, bolt.new_instance(base_class_path), bolt.parallelism)
117
+ declarer.addConfigurations(bolt.config)
115
118
  bolt.define_grouping(declarer)
116
119
  end
117
120
 
@@ -173,4 +176,4 @@ module RedStorm
173
176
  camel_case.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase!
174
177
  end
175
178
  end
176
- end
179
+ end
@@ -10,7 +10,11 @@ rescue LoadError
10
10
  require 'red_storm'
11
11
  end
12
12
 
13
- java_import 'backtype.storm.Config'
13
+ # see https://github.com/colinsurprenant/redstorm/issues/7
14
+ module Backtype
15
+ java_import 'backtype.storm.Config'
16
+ end
17
+
14
18
  java_import 'backtype.storm.LocalCluster'
15
19
  java_import 'backtype.storm.StormSubmitter'
16
20
  java_import 'backtype.storm.topology.TopologyBuilder'
@@ -36,7 +40,7 @@ class TopologyLauncher
36
40
  env = args[0].to_sym
37
41
  class_path = args[1]
38
42
 
39
- require class_path
43
+ require "./#{class_path}" # ./ for 1.9 compatibility
40
44
 
41
45
  topology_name = RedStorm::Configuration.topology_class.respond_to?(:topology_name) ? "/#{RedStorm::Configuration.topology_class.topology_name}" : ''
42
46
  puts("RedStorm v#{RedStorm::VERSION} starting topology #{RedStorm::Configuration.topology_class.name}#{topology_name} in #{env.to_s} environment")
@@ -1,3 +1,3 @@
1
1
  module RedStorm
2
- VERSION = '0.4.0'
2
+ VERSION = '0.5.0'
3
3
  end
@@ -9,6 +9,10 @@ rescue LoadError
9
9
  require 'red_storm'
10
10
  end
11
11
 
12
+ INSTALL_STORM_VERSION = "0.7.1"
13
+ INSTALL_JRUBY_VERSION = "1.6.7"
14
+ DEFAULT_GEMFILE = "Gemfile"
15
+
12
16
  CWD = Dir.pwd
13
17
  TARGET_DIR = "#{CWD}/target"
14
18
  TARGET_SRC_DIR = "#{TARGET_DIR}/src"
@@ -26,8 +30,9 @@ SRC_EXAMPLES = "#{RedStorm::REDSTORM_HOME}/examples"
26
30
  DST_EXAMPLES = "#{CWD}/examples"
27
31
 
28
32
  task :launch, :env, :class_file do |t, args|
33
+ version_token = RedStorm::RUNTIME['RUBY_VERSION'] == "--1.9" ? "RUBY1_9" : "RUBY1_8"
29
34
  gem_home = ENV["GEM_HOME"].to_s.empty? ? " -Djruby.gem.home=`gem env home`" : ""
30
- command = "java -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}"
35
+ command = "java -Djruby.compat.version=#{version_token} -cp \"#{TARGET_CLASSES_DIR}:#{TARGET_DEPENDENCY_DIR}/*\"#{gem_home} redstorm.TopologyLauncher #{args[:env]} #{args[:class_file]}"
31
36
  puts("launching #{command}")
32
37
  system(command)
33
38
  end
@@ -40,53 +45,46 @@ task :clean_jar do
40
45
  ant.delete :file => TARGET_CLUSTER_JAR
41
46
  end
42
47
 
43
- task :setup do
48
+ task :setup do
49
+ puts("\n--> Setting up target directories")
44
50
  ant.mkdir :dir => TARGET_DIR
45
51
  ant.mkdir :dir => TARGET_CLASSES_DIR
46
52
  ant.mkdir :dir => TARGET_SRC_DIR
47
53
  ant.mkdir :dir => TARGET_GEMS_DIR
54
+ ant.mkdir :dir => "#{TARGET_GEMS_DIR}/gems"
55
+ ant.mkdir :dir => "#{TARGET_GEMS_DIR}/bundler"
48
56
  ant.path :id => 'classpath' do
49
57
  fileset :dir => TARGET_DEPENDENCY_DIR
50
58
  fileset :dir => TARGET_CLASSES_DIR
51
59
  end
52
60
  end
53
61
 
54
- task :install => [:deps, :build] do
62
+ task :install => [:deps, :build, :gems] do
55
63
  puts("\nRedStorm install completed. All dependencies installed in #{TARGET_DIR}")
56
64
  end
57
65
 
58
66
  task :unpack do
59
- system("rmvn dependency:unpack -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
67
+ system("rmvn dependency:unpack \
68
+ -f #{RedStorm::REDSTORM_HOME}/pom.xml \
69
+ -DoutputDirectory=#{TARGET_DEPENDENCY_UNPACKED_DIR} \
70
+ -DmarkersDirectory=#{TARGET_MARKERS_DIR} \
71
+ -Dstorm-storm.version=#{INSTALL_STORM_VERSION} \
72
+ -Dorg.jruby-jruby-complete.version=#{INSTALL_JRUBY_VERSION}")
60
73
  end
61
74
 
62
- task :devjar => [:unpack, :clean_jar] do
75
+ task :jar, [:include_dir] => [:unpack, :clean_jar] do |t, args|
63
76
  ant.jar :destfile => TARGET_CLUSTER_JAR do
64
77
  fileset :dir => TARGET_CLASSES_DIR
65
78
  fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
66
- fileset :dir => TARGET_GEMS_DIR
67
- fileset :dir => RedStorm::REDSTORM_HOME do
68
- include :name => "examples/**/*"
79
+ fileset :dir => TARGET_GEMS_DIR do
80
+ # remove bundler config dir to avoid setting BUNDLE_PATH
81
+ exclude :name => "bundler/.bundle/**"
69
82
  end
70
83
  fileset :dir => JRUBY_SRC_DIR do
71
84
  exclude :name => "tasks/**"
72
85
  end
73
- manifest do
74
- attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
75
- end
76
- end
77
- puts("\nRedStorm generated dev jar file #{TARGET_CLUSTER_JAR}")
78
- end
79
-
80
- task :jar, [:dir] => [:unpack, :clean_jar] do |t, args|
81
- ant.jar :destfile => TARGET_CLUSTER_JAR do
82
- fileset :dir => TARGET_CLASSES_DIR
83
- fileset :dir => TARGET_DEPENDENCY_UNPACKED_DIR
84
- fileset :dir => TARGET_GEMS_DIR
85
- fileset :dir => JRUBY_SRC_DIR do
86
- exclude :name => "tasks/**"
87
- end
88
86
  fileset :dir => CWD do
89
- include :name => "#{args[:dir]}/**/*"
87
+ args[:include_dir].split(":").each{|dir| include :name => "#{dir}/**/*"}
90
88
  end
91
89
  manifest do
92
90
  attribute :name => "Main-Class", :value => "redstorm.TopologyLauncher"
@@ -111,8 +109,14 @@ task :examples do
111
109
  puts("\nRedStorm examples completed. All examples copied in #{DST_EXAMPLES}")
112
110
  end
113
111
 
114
- task :deps do
115
- system("rmvn dependency:copy-dependencies -f #{RedStorm::REDSTORM_HOME}/pom.xml -DoutputDirectory=#{TARGET_DEPENDENCY_DIR} -DmarkersDirectory=#{TARGET_MARKERS_DIR}")
112
+ task :deps => :setup do
113
+ puts("\n--> Installing dependencies")
114
+ system("rmvn dependency:copy-dependencies \
115
+ -f #{RedStorm::REDSTORM_HOME}/pom.xml \
116
+ -DoutputDirectory=#{TARGET_DEPENDENCY_DIR} \
117
+ -DmarkersDirectory=#{TARGET_MARKERS_DIR} \
118
+ -Dstorm-storm.version=#{INSTALL_STORM_VERSION} \
119
+ -Dorg.jruby-jruby-complete.version=#{INSTALL_JRUBY_VERSION}")
116
120
  end
117
121
 
118
122
  task :build => :setup do
@@ -130,7 +134,31 @@ task :build => :setup do
130
134
 
131
135
  # compile the JRuby proxy classes
132
136
  build_java_dir("#{TARGET_SRC_DIR}")
133
- end
137
+ end
138
+
139
+ task :gems, [:bundler_options] => :setup do |t, args|
140
+ bundler_options = args[:bundler_options].to_s.split(":").join(" ")
141
+
142
+ # basically copy original Gemfile to target/gems/bundler and install into this dir
143
+ gemfile = bundler_options =~ /--gemfile\s+([^\s]+)/ ? $1 : DEFAULT_GEMFILE
144
+ if bundler_options =~ /--gemfile\s+[^\s]+/
145
+ bundler_options.gsub!(/--gemfile\s+[^\s]+/, "--gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile")
146
+ else
147
+ bundler_options = bundler_options + " --gemfile #{TARGET_GEMS_DIR}/bundler/Gemfile"
148
+ end
149
+
150
+ puts("\n--> Installing gems in #{TARGET_GEMS_DIR}/gems")
151
+ system("gem install bundler --install-dir #{TARGET_GEMS_DIR}/gems --no-ri --no-rdoc --quiet --no-verbose")
152
+ system("gem install rake --install-dir #{TARGET_GEMS_DIR}/gems --no-ri --no-rdoc --quiet --no-verbose")
153
+
154
+ if File.exist?(gemfile)
155
+ puts("\n--> Bundling gems in #{TARGET_GEMS_DIR}/bundler using #{gemfile}")
156
+ system("cp #{gemfile} #{TARGET_GEMS_DIR}/bundler/Gemfile")
157
+ system("jruby #{RedStorm::RUNTIME['RUBY_VERSION']} -S bundle install #{bundler_options} --path #{TARGET_GEMS_DIR}/bundler/")
158
+ elsif gemfile != DEFAULT_GEMFILE
159
+ puts("WARNING: #{gemfile} not found, cannot bundle gems")
160
+ end
161
+ end
134
162
 
135
163
  def build_java_dir(source_folder)
136
164
  puts("\n--> Compiling Java")
@@ -144,10 +172,12 @@ def build_java_dir(source_folder)
144
172
  :includeantruntime => "no",
145
173
  :verbose => false,
146
174
  :listfiles => true
147
- )
175
+ ) do
176
+ # compilerarg :value => "-Xlint:unchecked"
177
+ end
148
178
  end
149
179
 
150
180
  def build_jruby(source_path)
151
181
  puts("\n--> Compiling JRuby")
152
- system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-0.6.2.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
182
+ system("cd #{RedStorm::REDSTORM_HOME}; jrubyc -t #{TARGET_SRC_DIR} --verbose --java -c \"#{TARGET_DEPENDENCY_DIR}/storm-#{INSTALL_STORM_VERSION}.jar\" -c \"#{TARGET_CLASSES_DIR}\" #{source_path}")
153
183
  end
data/pom.xml CHANGED
@@ -2,10 +2,11 @@
2
2
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3
3
  <modelVersion>4.0.0</modelVersion>
4
4
 
5
+
5
6
  <groupId>redstorm</groupId>
6
7
  <artifactId>redstorm</artifactId>
7
- <version>0.4.0</version>
8
- <name>RedStorm JRuby on Storm</name>
8
+ <version>0.5.0</version>
9
+ <name>RedStorm</name>
9
10
 
10
11
  <properties>
11
12
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -27,13 +28,13 @@
27
28
  <dependency>
28
29
  <groupId>storm</groupId>
29
30
  <artifactId>storm</artifactId>
30
- <version>0.6.2</version>
31
+ <version>${storm-storm.version}</version>
31
32
  </dependency>
32
33
 
33
34
  <dependency>
34
35
  <groupId>org.jruby</groupId>
35
36
  <artifactId>jruby-complete</artifactId>
36
- <version>1.6.6</version>
37
+ <version>${org.jruby-jruby-complete.version}</version>
37
38
  </dependency>
38
39
  </dependencies>
39
40
 
@@ -48,7 +49,7 @@
48
49
  <artifactItem>
49
50
  <groupId>org.jruby</groupId>
50
51
  <artifactId>jruby-complete</artifactId>
51
- <version>1.6.6</version>
52
+ <version>${org.jruby-jruby-complete.version}</version>
52
53
  <type>jar</type>
53
54
  <overWrite>false</overWrite>
54
55
  </artifactItem>
@@ -58,6 +58,16 @@ public class JRubyBolt implements IRichBolt {
58
58
  bolt.declareOutputFields(declarer);
59
59
  }
60
60
 
61
+ @Override
62
+ public Map<String, Object> getComponentConfiguration() {
63
+ // getComponentConfiguration is executed in the topology creation time, before serialisation.
64
+ // do not set the _proxyBolt instance variable here to avoid JRuby serialization
65
+ // issues. Just create tmp bolt instance to call declareOutputFields.
66
+ IRichBolt bolt = newProxyBolt(_baseClassPath, _realBoltClassName);
67
+ return bolt.getComponentConfiguration();
68
+ }
69
+
70
+
61
71
  private static IRichBolt newProxyBolt(String baseClassPath, String realBoltClassName) {
62
72
  try {
63
73
  redstorm.proxy.Bolt proxy = new redstorm.proxy.Bolt(baseClassPath, realBoltClassName);
@@ -33,15 +33,6 @@ public class JRubySpout implements IRichSpout {
33
33
  _realSpoutClassName = realSpoutClassName;
34
34
  }
35
35
 
36
- @Override
37
- public boolean isDistributed() {
38
- // isDistributed is executed in the topology creation time before serialisation.
39
- // do not set the _proxySpout instance variable here to avoid JRuby serialization
40
- // issues. Just create tmp spout instance to call isDistributed.
41
- IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
42
- return spout.isDistributed();
43
- }
44
-
45
36
  @Override
46
37
  public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
47
38
  // create instance of the jruby proxy class here, after deserialization in the workers.
@@ -54,6 +45,16 @@ public class JRubySpout implements IRichSpout {
54
45
  _proxySpout.close();
55
46
  }
56
47
 
48
+ @Override
49
+ public void activate() {
50
+ _proxySpout.activate();
51
+ }
52
+
53
+ @Override
54
+ public void deactivate() {
55
+ _proxySpout.deactivate();
56
+ }
57
+
57
58
  @Override
58
59
  public void nextTuple() {
59
60
  _proxySpout.nextTuple();
@@ -78,6 +79,15 @@ public class JRubySpout implements IRichSpout {
78
79
  spout.declareOutputFields(declarer);
79
80
  }
80
81
 
82
+ @Override
83
+ public Map<String, Object> getComponentConfiguration() {
84
+ // getComponentConfiguration is executed in the topology creation time before serialisation.
85
+ // do not set the _proxySpout instance variable here to avoid JRuby serialization
86
+ // issues. Just create tmp spout instance to call declareOutputFields.
87
+ IRichSpout spout = newProxySpout(_baseClassPath, _realSpoutClassName);
88
+ return spout.getComponentConfiguration();
89
+ }
90
+
81
91
  private static IRichSpout newProxySpout(String baseClassPath, String realSpoutClassName) {
82
92
  try {
83
93
  redstorm.proxy.Spout proxy = new redstorm.proxy.Spout(baseClassPath, realSpoutClassName);
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: redstorm
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.4.0
5
+ version: 0.5.0
6
6
  platform: ruby
7
7
  authors:
8
8
  - Colin Surprenant
@@ -10,17 +10,17 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2012-02-08 00:00:00 Z
13
+ date: 2012-06-01 00:00:00 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
- name: rubyforge
16
+ name: rspec
17
17
  prerelease: false
18
18
  requirement: &id001 !ruby/object:Gem::Requirement
19
19
  none: false
20
20
  requirements:
21
- - - ">="
21
+ - - ~>
22
22
  - !ruby/object:Gem::Version
23
- version: "0"
23
+ version: 2.8.0
24
24
  type: :development
25
25
  version_requirements: *id001
26
26
  - !ruby/object:Gem::Dependency
@@ -29,9 +29,9 @@ dependencies:
29
29
  requirement: &id002 !ruby/object:Gem::Requirement
30
30
  none: false
31
31
  requirements:
32
- - - ~>
32
+ - - ">="
33
33
  - !ruby/object:Gem::Version
34
- version: 0.9.2
34
+ version: "0"
35
35
  type: :runtime
36
36
  version_requirements: *id002
37
37
  - !ruby/object:Gem::Dependency
@@ -58,6 +58,8 @@ files:
58
58
  - lib/red_storm.rb
59
59
  - lib/red_storm/application.rb
60
60
  - lib/red_storm/configuration.rb
61
+ - lib/red_storm/configurator.rb
62
+ - lib/red_storm/loggable.rb
61
63
  - lib/red_storm/simple_bolt.rb
62
64
  - lib/red_storm/simple_spout.rb
63
65
  - lib/red_storm/simple_topology.rb
@@ -78,8 +80,10 @@ files:
78
80
  - examples/simple/exclamation_bolt.rb
79
81
  - examples/simple/exclamation_topology.rb
80
82
  - examples/simple/exclamation_topology2.rb
83
+ - examples/simple/Gemfile
81
84
  - examples/simple/random_sentence_spout.rb
82
85
  - examples/simple/redis_word_count_topology.rb
86
+ - examples/simple/ruby_version_topology.rb
83
87
  - examples/simple/split_sentence_bolt.rb
84
88
  - examples/simple/word_count_bolt.rb
85
89
  - examples/simple/word_count_topology.rb
@@ -111,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
111
115
  requirements:
112
116
  - - ">="
113
117
  - !ruby/object:Gem::Version
114
- version: 1.3.0
118
+ version: "0"
115
119
  requirements: []
116
120
 
117
121
  rubyforge_project: redstorm