redstorm 0.6.6 → 0.7.0.beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d7b22ba82aa7cc0135889207e53253b16c129efb
+  data.tar.gz: ae678b9666da95da30f4f2a23a344d0fd44df022
+SHA512:
+  metadata.gz: 1bb9e682e7b053f7a2df1c043ab5821df427cf62bfcd26e1281253574f097e88f623bc93db9e73ff1494c865ea2ea3e61e9c7004a0b6c17819dd5f903676f1cc
+  data.tar.gz: bb6148070c8de87f56339c517e6ed8eed4d873a35db5212dc98b720d929fdfe9ba960dd51273995bfee0b2d6a473e4399f7c8c0e0abbe54e2a6bf65d0bcd27f0

data/CHANGELOG.md

@@ -97,4 +97,9 @@
 - [issue #76](https://github.com/colinsurprenant/redstorm/issues/76) - avoid shelling out to storm jar command for cluster submission
 
 # 0.6.6, 07-25-2013
-- updated example Kafka topology for new dependencies for Storm KafkaSpout
+- updated example Kafka topology for new dependencies for Storm KafkaSpout
+
+# 0.7.0.beta1, 03-2014
+- refactored the proxy classes for better performance
+- Storm 0.9.1-incubating and JRuby 1.7.11
+- added Trident example in `examples/trident/`

data/README.md

@@ -1,6 +1,9 @@
 # RedStorm - JRuby on Storm
 
+[![Gem Version](https://badge.fury.io/rb/redstorm.png)](http://badge.fury.io/rb/redstorm)
 [![build status](https://secure.travis-ci.org/colinsurprenant/redstorm.png)](http://travis-ci.org/colinsurprenant/redstorm)
+[![Code Climate](https://codeclimate.com/github/colinsurprenant/redstorm.png)](https://codeclimate.com/github/colinsurprenant/redstorm)
+[![Coverage Status](https://coveralls.io/repos/colinsurprenant/redstorm/badge.png?branch=master)](https://coveralls.io/r/colinsurprenant/redstorm?branch=master)
 
 RedStorm provides a Ruby DSL using JRuby integration for the [Storm](https://github.com/nathanmarz/storm/) distributed realtime computation system.
 
@@ -13,17 +16,15 @@ Check also these related projects:
 
 ## Documentation
 
-<!--
----
-This is the documentation for the **current 0.6.6-beta2 version of RedStorm** - the **[latest released Gem is v0.6.5](https://github.com/colinsurprenant/redstorm/wiki/RedStorm-Gem-v0.6.5-Documentation)**
-
----
--->
 Chances are new versions of RedStorm will introduce changes that will break compatibility or change the developement workflow. To prevent out-of-sync documentation, per version specific documentation are kept [in the wiki](https://github.com/colinsurprenant/redstorm/wiki) when necessary.
 
 ## Dependencies
 
-Tested on **OSX 10.8.3** and **Ubuntu Linux 12.10** using **Storm 0.9.0-wip16** and **JRuby 1.7.4** and **OpenJDK 7**
+#### Stable 0.6.6
+- Tested on **OSX 10.8.3**, **Ubuntu Linux 12.10** using **Storm 0.9.0-wip16**, **JRuby 1.7.4**, **OpenJDK 7**
+
+#### Current 0.7.0.beta1
+- Tested on **OSX 10.9.1**, **Ubuntu Linux 12.10** using **Storm 0.9.1-incubating**, **JRuby 1.7.11**, **OpenJDK 7**
 
 ## Installation
 

data/examples/dsl/exclamation_topology.rb

@@ -24,10 +24,8 @@ module RedStorm
       configure do |env|
         debug false
         max_task_parallelism 4
-        if env == :cluster
-          num_workers 4
-          max_spout_pending(1000)
-        end
+        num_workers 1
+        max_spout_pending 1000
       end
 
       on_submit do |env|

data/examples/dsl/exclamation_topology2.rb

@@ -21,15 +21,14 @@ module RedStorm
 
       bolt ExclamationBolt, :id => :ExclamationBolt2, :parallelism => 2 do
         source ExclamationBolt, :shuffle
+        debug true
       end
 
       configure do |env|
-        debug true
+        debug false
         max_task_parallelism 4
-        if env == :cluster
-          num_workers 4
-          max_spout_pending(1000)
-        end
+        num_workers 1
+        max_spout_pending 1000
       end
 
       on_submit do |env|

data/examples/dsl/hello_world_topology.rb

@@ -19,4 +19,11 @@ class HelloWorldTopology < RedStorm::DSL::Topology
   bolt HelloWorldBolt do
     source HelloWorldSpout, :global
   end
+
+  configure do
+    debug false
+    max_task_parallelism 4
+    num_workers 1
+    max_spout_pending 1000
+  end
 end

data/examples/dsl/kafka_topology.rb

@@ -48,10 +48,14 @@ class KafkaTopology < RedStorm::DSL::Topology
   bolt SplitStringBolt do
     output_fields :word
     source KafkaSpout, :shuffle
+    debug true
   end
 
   configure do |env|
-    debug true
+    debug false
+    max_task_parallelism 4
+    num_workers 1
+    max_spout_pending 1000
   end
 
   on_submit do |env|

data/examples/dsl/redis_word_count_topology.rb

@@ -39,19 +39,15 @@ module RedStorm
       spout RedisWordSpout
 
       bolt WordCountBolt, :parallelism => 2 do
+        debug true
         source RedisWordSpout, :fields => ["word"]
       end
 
       configure do |env|
-        debug true
-        case env
-        when :local
-          max_task_parallelism 2
-        when :cluster
-          max_task_parallelism 2
-          num_workers 2
-          max_spout_pending(1000)
-        end
+        debug false
+        max_task_parallelism 2
+        num_workers 1
+        max_spout_pending 1000
       end
     end
   end

data/examples/dsl/ruby_version_topology.rb

@@ -24,6 +24,8 @@ module RedStorm
       spout VersionSpout
 
       configure do |env|
+        max_task_parallelism 1
+        num_workers 1
         debug false
       end
 

data/examples/dsl/word_count_topology.rb

@@ -13,16 +13,15 @@ module RedStorm
       end
 
       bolt WordCountBolt, :parallelism => 2 do
+        debug true
         source SplitSentenceBolt, :fields => ["word"]
       end
 
       configure :word_count do |env|
-        debug true
+        debug false
         max_task_parallelism 4
-        if env == :cluster
-          num_workers 6
-          max_spout_pending(1000)
-        end
+        num_workers 1
+        max_spout_pending 1000
       end
 
       on_submit do |env|

data/examples/trident/word_count_query.rb

@@ -0,0 +1,33 @@
+require "red_storm"
+require "json"
+
+java_import "backtype.storm.utils.DRPCClient"
+
+# Usage:
+#
+# This is a DRPC client that will query a Storm cluster trident drpc topology.
+# See the trident word_count_topology.rb for runnnig the drpc topology.
+#
+# Edit the host and port below.
+
+module Example
+
+  # this is not a topology, the redstorm topology_launcher will launch any class with the
+  # start method in the correct storm environment
+
+  class TridentWordCountQuery
+    RedStorm::Configuration.topology_class = self
+
+    def start(env)
+      puts("TridentWordCountQuery starting")
+
+      client = DRPCClient.new("localhost", 3772)
+      loop do
+        json_result = client.execute("words", "cat the dog jumped")
+        puts("DRPC execute=#{JSON.parse(json_result)[0][0]}")
+
+        sleep(2)
+      end
+    end
+  end
+end

data/examples/trident/word_count_topology.rb

@@ -0,0 +1,153 @@
+require 'red_storm'
+require 'json'
+
+java_import "backtype.storm.LocalCluster"
+java_import "backtype.storm.LocalDRPC"
+java_import "backtype.storm.StormSubmitter"
+java_import "backtype.storm.generated.StormTopology"
+java_import "backtype.storm.tuple.Fields"
+java_import "backtype.storm.tuple.Values"
+java_import "storm.trident.TridentState"
+java_import "storm.trident.TridentTopology"
+java_import "storm.trident.operation.BaseFunction"
+java_import "storm.trident.operation.TridentCollector"
+java_import "storm.trident.operation.builtin.Count"
+java_import "storm.trident.operation.builtin.FilterNull"
+java_import "storm.trident.operation.builtin.MapGet"
+java_import "storm.trident.operation.builtin.Sum"
+java_import "storm.trident.testing.FixedBatchSpout"
+java_import "storm.trident.testing.MemoryMapState"
+java_import "storm.trident.tuple.TridentTuple"
+
+java_import 'redstorm.storm.jruby.JRubyTridentFunction'
+
+REQUIRE_PATH = Pathname.new(__FILE__).relative_path_from(Pathname.new(RedStorm::BASE_PATH)).to_s
+
+# Usage:
+#
+# Local mode:
+#
+# $ redstorm install
+# $ redstorm examples
+# $ restorm local examples/trident/word_count_topology.rb
+#
+# Cluster mode:
+#
+# $ redstorm install
+# $ redstorm examples
+# $ redstorm jar examples
+# $ redstorm cluster examples/trident/word_count_topology.rb
+#
+#  After submission, wait a bit for topology to startup and launch the drpc query example:
+#  Edit word_count_query.rb to set the host/port of your cluster drpc daemon.
+#
+# $ redstorm local examples/trident/word_count_query.rb
+
+module Examples
+  class TridentSplit
+
+    def execute(tuple, collector)
+      tuple[0].split(" ").each do |word|
+        collector.emit(Values.new(word))
+      end
+    end
+
+    def prepare(conf, context); end
+    def cleanup;end
+  end
+
+  class TridentWordCountTopology
+    RedStorm::Configuration.topology_class = self
+
+    def build_topology(local_drpc)
+      spout = FixedBatchSpout.new(
+        Fields.new("sentence"), 3,
+        Values.new("the cow jumped over the moon"),
+        Values.new("the man went to the store and bought some candy"),
+        Values.new("four score and seven years ago"),
+        Values.new("how many apples can you eat"),
+        Values.new("to be or not to be the person")
+      )
+      spout.cycle = true
+
+      topology = TridentTopology.new
+
+      stream = topology.new_stream("spout1", spout)
+        .parallelism_hint(3)
+        .each(
+          Fields.new("sentence"),
+          JRubyTridentFunction.new(REQUIRE_PATH, "Examples::TridentSplit"),
+          Fields.new("word")
+        )
+        .groupBy(
+          Fields.new("word")
+        )
+        .persistentAggregate(
+          MemoryMapState::Factory.new,
+          Count.new,
+          Fields.new("count")
+        )
+        .parallelism_hint(3)
+
+      # topology.newDRPCStream("words", drpc)
+      topology.newDRPCStream("words", local_drpc)
+        .each(
+          Fields.new("args"),
+          JRubyTridentFunction.new(REQUIRE_PATH, "Examples::TridentSplit"),
+          Fields.new("word")
+        )
+        .groupBy(
+          Fields.new("word")
+        )
+        .stateQuery(
+          stream,
+          Fields.new("word"),
+          MapGet.new,
+          Fields.new("count")
+        )
+        .each(
+          Fields.new("count"),
+          FilterNull.new
+        )
+        .aggregate(
+          Fields.new("count"),
+          Sum.new,
+          Fields.new("sum")
+        )
+
+      topology.build
+    end
+
+    def display_drpc(client)
+      loop do
+        sleep(2)
+
+        json_result = client.execute("words", "cat the dog jumped")
+        puts("DRPC execute=#{JSON.parse(json_result)[0][0]}")
+      end
+    end
+
+    def start(env)
+      conf = Backtype::Config.new
+      conf.debug = false
+      conf.max_spout_pending = 20
+
+      case env
+      when :local
+        local_drpc = LocalDRPC.new
+        submitter = LocalCluster.new
+        conf.num_workers = 1 # set to 1 in local, see https://issues.apache.org/jira/browse/STORM-113
+      when :cluster
+        local_drpc = nil
+        submitter = StormSubmitter
+        conf.put("drpc.servers", ["localhost"])
+        conf.num_workers = 3
+      end
+
+      submitter.submit_topology("trident_word_count", conf, build_topology(local_drpc));
+
+      display_drpc(local_drpc) if local_drpc
+    end
+  end
+
+end

data/ivy/storm_dependencies.xml

@@ -2,7 +2,7 @@
 <ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="redstorm" module="storm-deps"/>
   <dependencies>
-    <dependency org="storm" name="storm" rev="0.9.0-wip16" conf="default" transitive="true" />
+    <dependency org="org.apache.storm" name="storm-core" rev="0.9.1-incubating" conf="default" transitive="true" />
     <override org="org.slf4j" module="slf4j-log4j12" rev="1.6.3"/>
   </dependencies>
 </ivy-module>

data/ivy/topology_dependencies.xml

@@ -2,11 +2,12 @@
 <ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="redstorm" module="topology-deps"/>
   <dependencies>
-    <dependency org="org.jruby" name="jruby-core" rev="1.7.4" conf="default" transitive="true"/>
+    <dependency org="org.jruby" name="jruby-core" rev="1.7.11" conf="default" transitive="true"/>
+    <dependency org="org.jruby" name="jruby-stdlib" rev="1.7.11" conf="default" transitive="true"/>
 
     <!-- explicitely specify jffi to also fetch the native jar. make sure to update jffi version matching jruby-core version -->
     <!-- this is the only way I found using Ivy to fetch the native jar -->
-    <dependency org="com.github.jnr" name="jffi" rev="1.2.5" conf="default" transitive="true">
+    <dependency org="com.github.jnr" name="jffi" rev="1.2.7" conf="default" transitive="true">
       <artifact name="jffi" type="jar" />
       <artifact name="jffi" type="jar" m:classifier="native"/>
     </dependency>

data/lib/red_storm.rb

@@ -1,10 +1,13 @@
-require 'rubygems'
-
 require 'red_storm/version'
 require 'red_storm/environment'
 require 'red_storm/configuration'
+require 'red_storm/configurator'
 require 'red_storm/dsl/bolt'
+require 'red_storm/dsl/batch_bolt'
+require 'red_storm/dsl/batch_committer_bolt'
 require 'red_storm/dsl/spout'
+require 'red_storm/dsl/batch_spout'
 require 'red_storm/dsl/topology'
 require 'red_storm/dsl/drpc_topology'
 require 'red_storm/dsl/tuple'
+require 'red_storm/dsl/output_collector'

data/lib/red_storm/configurator.rb

@@ -1,3 +1,15 @@
+# This hack get rif of the "Use RbConfig instead of obsolete and deprecated Config"
+# deprecation warning that is triggered by "java_import 'backtype.storm.Config'".
+begin
+  Object.send :remove_const, :Config
+  Config = RbConfig
+rescue NameError
+end
+
+module Backtype
+  java_import 'backtype.storm.Config'
+end
+
 module RedStorm
 
   class Configurator

data/lib/red_storm/dsl/batch_bolt.rb

@@ -0,0 +1,34 @@
+module RedStorm
+  module DSL
+
+    class BatchBolt < Bolt
+      attr_reader :id
+
+      def self.java_proxy; "Java::RedstormStormJruby::JRubyBatchBolt"; end
+
+      def self.on_finish_batch(method_name = nil, &on_finish_batch_block)
+        body = block_given? ? on_finish_batch_block : lambda {self.send((method_name || :on_finish_batch).to_sym)}
+        define_method(:on_finish_batch, body)
+      end
+
+      def prepare(config, context, collector, id)
+        @collector = collector
+        @context = context
+        @config = config
+        @id = id
+
+        on_init
+      end
+
+      def finish_batch
+        on_finish_batch
+      end
+
+      private
+
+      # default noop optional dsl callbacks
+      def on_finish_batch; end
+
+    end
+  end
+end