wukong-storm 0.1.1 → 0.2.0

data/src/main/java/com/infochimps/wukong/storm/StateBuilder.java

@@ -0,0 +1,46 @@
+package com.infochimps.wukong.storm;
+
+import org.apache.log4j.Logger;
+import com.infochimps.storm.trident.KafkaState;
+
+public class StateBuilder extends Builder {
+
+    static Logger LOG = Logger.getLogger(StateBuilder.class);
+
+    public KafkaState.Factory state() {
+	return new KafkaState.Factory(kafkaOutputTopic(), zookeeperHosts());
+    }
+
+    public KafkaState.Updater updater() {
+	return new KafkaState.Updater();
+    }
+
+    @Override
+    public Boolean valid() {
+	if (kafkaOutputTopic() == null) {
+	    LOG.error("Must set a Kafka output topic using the " + KAFKA_OUTPUT_TOPIC + "property");
+	    return false;
+	}
+	return true;
+    }
+
+    @Override
+    public void logInfo() {
+	LOG.info("STATE: Writing to Kafka topic <" + kafkaOutputTopic() + ">");
+    }
+
+    public static String usage() {
+	String s = "STATE OPTIONS\n"
+	    + "\n"
+	    + "The only available state is Kafka which has the following options:\n"
+	    + "\n"
+	    + "  " + String.format("%10s", KAFKA_OUTPUT_TOPIC) + "  The Kafka output topic (Required)\n";
+	return s;
+    }
+    
+    public static String KAFKA_OUTPUT_TOPIC = "wukong.output.kafka.topic";
+    public String kafkaOutputTopic() {
+	return prop(KAFKA_OUTPUT_TOPIC);
+    }
+    
+}

data/src/main/java/com/infochimps/wukong/storm/TopologyBuilder.java

@@ -0,0 +1,130 @@
+package com.infochimps.wukong.storm;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Arrays;
+
+import org.apache.log4j.Logger;
+
+import backtype.storm.generated.StormTopology;
+import backtype.storm.tuple.Fields;
+
+import storm.trident.Stream;
+import storm.trident.TridentTopology;
+
+import storm.trident.operation.BaseFunction;
+import storm.trident.operation.TridentCollector;
+import backtype.storm.tuple.Values;
+import storm.trident.tuple.TridentTuple;
+
+public class TopologyBuilder extends Builder {
+
+    private static class CombineMetadata extends BaseFunction {
+        @Override
+        public void execute(TridentTuple tuple, TridentCollector collector) {
+	    String  content    = tuple.getStringByField("content");
+            String  metadata   = tuple.getStringByField("metadata");
+            Integer lineNumber = tuple.getIntegerByField("linenumber");
+	    LOG.debug(String.format("%s\t%s\t%s", metadata, content, lineNumber));
+            collector.emit(new Values(String.format("%s\t%s\t%s", metadata, content, lineNumber)));
+        }
+    }
+
+    private SpoutBuilder    spoutBuilder;
+    private DataflowBuilder dataflowBuilder;
+    private StateBuilder    stateBuilder;
+
+    static Logger LOG = Logger.getLogger(TopologyBuilder.class);
+
+    public TopologyBuilder() {
+	this.spoutBuilder    = new SpoutBuilder();
+	this.dataflowBuilder = new DataflowBuilder(spoutBuilder);
+	this.stateBuilder    = new StateBuilder();
+    }
+
+    @Override
+    public Boolean valid() {
+	if (topologyName() == null) {
+	    LOG.error("Must set a topology name using the " + TOPOLOGY_NAME + " property");
+	    return false;
+	}
+	if (!spoutBuilder.valid()) { return false; }
+	if (!dataflowBuilder.valid()) { return false; }
+	if (!stateBuilder.valid()) { return false; }
+	return true;
+    }
+
+    @Override
+    public void logInfo() {
+	LOG.info("\n");
+	spoutBuilder.logInfo();
+	dataflowBuilder.logInfo();
+	stateBuilder.logInfo();
+    }
+    
+    public StormTopology topology() {
+	TridentTopology top = new TridentTopology();
+
+	Stream spoutOutput = top.newStream(topologyName(), spoutBuilder.spout())
+	    .parallelismHint(spoutBuilder.inputParallelism());
+
+	Stream possiblyShuffledSpoutOutput;
+	if (needToShuffleSpoutOutput()) {
+	    possiblyShuffledSpoutOutput = spoutOutput.shuffle();
+	} else {
+	    possiblyShuffledSpoutOutput = spoutOutput;
+	}
+
+	Stream dataflowInput;
+	if (spoutBuilder.isBlobSpout()) {
+	    dataflowInput = possiblyShuffledSpoutOutput.each(new Fields("content", "metadata", "linenumber"), new CombineMetadata(), new Fields("str"));
+	} else {
+	    dataflowInput = possiblyShuffledSpoutOutput;
+	}
+	
+	Stream dataflowOutput = dataflowInput.each(new Fields("str"), dataflowBuilder.dataflow(), new Fields("_wukong"))
+	    .parallelismHint(dataflowBuilder.dataflowParallelism());
+
+	dataflowOutput.partitionPersist(stateBuilder.state(), new Fields("_wukong"), stateBuilder.updater());
+	
+	return top.build();
+    }
+
+    public static String usage() {
+	String s = "\n"
+	    + "Dynamically assemble and launch a parametrized Storm topology that\n"
+	    + "embeds Wukong dataflow(s).  The current overall \"shape\" of the\n"
+	    + "topology is\n"
+	    + "\n"
+	    + "  spout -> wukong dataflow -> state\n"
+	    + "\n"
+	    + "The available spouts read from Kafka or S3.  The only available state\n"
+	    + "is Kafka.\n"
+	    + "\n"
+	    + "TOPOLOGY OPTIONS\n"
+	    + "\n"
+	    + "The following options can be used for any topology:\n"
+	    + "\n"
+	    + "  " + String.format("%10s", TOPOLOGY_NAME) + "  Name of the Storm topology that will be launched  (Required)\n"
+	    + "  " + String.format("%10s", KAFKA_HOSTS) + "  Comma-separated list of Kafka host (and optional port) pairs  (Default: " + DEFAULT_KAFKA_HOSTS + ")\n"
+	    + "  " + String.format("%10s", ZOOKEEPER_HOSTS) + "  Comma-separated list of Zookeeper host (and optional port) pairs  (Default: " + DEFAULT_ZOOKEEPER_HOSTS + ")\n"
+	    + "\n"
+	    + SpoutBuilder.usage()
+	    + "\n"
+	    + DataflowBuilder.usage()
+	    + "\n"
+	    + StateBuilder.usage()
+	    + "\n";
+	return s;
+    }
+
+    private Boolean needToShuffleSpoutOutput() {
+	return (dataflowBuilder.dataflowParallelism() > spoutBuilder.inputParallelism());
+    }
+
+    public static String TOPOLOGY_NAME                  = "wukong.topology";
+    public String topologyName() {
+	return prop(TOPOLOGY_NAME);
+    }
+    
+}

data/src/main/java/com/infochimps/wukong/storm/TopologySubmitter.java

@@ -0,0 +1,181 @@
+package com.infochimps.wukong.storm;
+
+import java.io.File;
+
+import org.apache.log4j.Logger;
+
+import backtype.storm.Config;
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.AlreadyAliveException;
+import backtype.storm.generated.InvalidTopologyException;
+
+import com.infochimps.wukong.storm.TopologyBuilder;
+
+public class TopologySubmitter {
+
+    private static Logger LOG = Logger.getLogger(TopologySubmitter.class);
+    
+    private TopologyBuilder builder;
+    private Config          config;
+
+    public static void main(String[] args) throws Exception {
+	setPropertiesFromArgsBecauseStupidlyHard(args);
+	TopologySubmitter submitter = new TopologySubmitter();
+	submitter.setConfig();
+	submitter.validate();
+	submitter.submit();
+	System.exit(0);
+    }
+
+    public static void setPropertiesFromArgsBecauseStupidlyHard(String[] args) {
+	int numArgs      = args.length;
+	int argIndex     = 0;
+	boolean isOption = false;
+	while (argIndex < numArgs) {
+	    String arg = args[argIndex];
+	    if (isOption) {
+		setPropertyFromArgBecauseStupidlyHard(arg);
+		isOption = false;
+	    } else {
+		if (arg.matches("-D.+")) {
+		    setPropertyFromArgBecauseStupidlyHard(arg.substring(2));
+		} else if (arg.matches("-D")) {
+		    isOption = true;
+		} else {
+		    LOG.error("Malformed option: " + arg);
+		}
+	    }
+	    argIndex += 1;
+	}
+    }
+
+    private static void setPropertyFromArgBecauseStupidlyHard(String arg) {
+	String[] parts = arg.split("=");
+	if (parts.length >= 2) {
+	    String key   = parts[0];
+	    String value = arg.substring(key.length() + 1);
+	    System.setProperty(key, value);
+	} else {
+	    LOG.error("Invalid property: " + arg);
+	}
+    }
+
+    private String prop(String key, String defaultValue) {
+	if (System.getProperty(key) == null) {
+	    System.setProperty(key, defaultValue);
+	}
+	return prop(key);
+    }
+
+    private String prop(String key) {
+	return System.getProperty(key);
+    }
+    
+    public TopologySubmitter() {
+	this.builder = new TopologyBuilder();
+	this.config  = new Config();
+    }
+
+    private void validate() {
+	if (!builder.valid()) {
+	    System.out.println(usage());
+	    System.exit(1);
+	}
+    }
+
+    public String usage() {
+	return "usage: storm jar " + fullyQualifiedClassPath() + " -DOPTION=VALUE ..." + TopologyBuilder.usage();
+    }
+    
+    public File fullyQualifiedClassPath() {
+	return new File(TopologySubmitter.class.getProtectionDomain().getCodeSource().getLocation().getPath());
+    }
+    
+    public void setConfig() {
+	setDebug();
+	setMaxSpoutPending();
+	setMaxTaskParallelism();
+	setMessageTimeoutSecs();
+	setNumAckers();
+	setNumWorkers();
+	setOptimize();
+	setStatsSampleRate();
+    }
+
+    public void submit() {
+	try {
+	    builder.logInfo();
+	    StormSubmitter.submitTopology(builder.topologyName(), config, builder.topology());
+	} catch (AlreadyAliveException e) {
+	    LOG.error("Topology " + builder.topologyName() + " is already running", e);
+	    System.exit(2);
+	} catch (InvalidTopologyException e) {
+	    LOG.error("Topology " + builder.topologyName() + " is invalid", e);
+	    System.exit(3);
+	}
+    }
+
+    public void setDebug() {
+	String value = prop(Config.TOPOLOGY_DEBUG);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_DEBUG + " to " + value);
+	    config.setDebug(Boolean.parseBoolean(value));
+	}
+    }
+
+    public void setMaxSpoutPending() {
+	String value = prop(Config.TOPOLOGY_MAX_SPOUT_PENDING);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_MAX_SPOUT_PENDING + " to " + value);
+	    config.setMaxSpoutPending(Integer.parseInt(value));
+	}
+    }
+
+    public void setMaxTaskParallelism() {
+	String value = prop(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_MAX_TASK_PARALLELISM + " to " + value);
+	    config.setMaxTaskParallelism(Integer.parseInt(value));
+	}
+    }
+
+    public void setMessageTimeoutSecs() {
+	String value = prop(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + " to " + value);
+	    config.setMessageTimeoutSecs(Integer.parseInt(value));
+	}
+    }
+
+    public void setNumAckers() {
+	String value = prop(Config.TOPOLOGY_ACKER_EXECUTORS);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_ACKER_EXECUTORS + " to " + value);
+	    config.setNumAckers(Integer.parseInt(value));
+	}
+    }
+
+    public void setNumWorkers() {
+	String value = prop(Config.TOPOLOGY_WORKERS);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_WORKERS + " to " + value);
+	    config.setNumWorkers(Integer.parseInt(value));
+	}
+    }
+
+    public void setOptimize() {
+	String value = prop(Config.TOPOLOGY_OPTIMIZE);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_OPTIMIZE + " to " + value);
+	    config.setDebug(Boolean.parseBoolean(value));
+	}
+    }
+
+    public void setStatsSampleRate() {
+	String value = prop(Config.TOPOLOGY_STATS_SAMPLE_RATE);
+	if (! (value == null)) {
+	    LOG.info("Setting " + Config.TOPOLOGY_STATS_SAMPLE_RATE + " to " + value);
+	    config.setStatsSampleRate(Integer.parseInt(value));
+	}
+    }
+}

data/wukong-storm.gemspec

@@ -14,9 +14,10 @@ Gem::Specification.new do |gem|
 EOF
 
   gem.files         = `git ls-files`.split("\n")
-  gem.executables   = ['wu-storm']
+  gem.executables   = ['wu-storm', 'wu-bolt']
   gem.test_files    = gem.files.grep(/^spec/)
   gem.require_paths = ['lib']
 
-  gem.add_dependency('wukong', '3.0.1')
+  gem.add_dependency('wukong', '4.0.0')
+  gem.add_dependency('kafka-rb')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wukong-storm
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wukong
@@ -19,7 +19,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,11 +27,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: kafka-rb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email: coders@infochimps.org
 executables:
 - wu-storm
+- wu-bolt
 extensions: []
 extra_rdoc_files: []
 files:
@@ -42,15 +59,31 @@ files:
 - LICENSE.md
 - README.md
 - Rakefile
+- bin/wu-bolt
 - bin/wu-storm
 - lib/wukong-storm.rb
-- lib/wukong-storm/driver.rb
-- lib/wukong-storm/runner.rb
+- lib/wukong-storm/bolt_driver.rb
+- lib/wukong-storm/bolt_runner.rb
+- lib/wukong-storm/storm_invocation.rb
+- lib/wukong-storm/storm_runner.rb
 - lib/wukong-storm/storm_settings.java
 - lib/wukong-storm/version.rb
+- lib/wukong-storm/wukong-storm.jar
+- pom.xml
 - spec/spec_helper.rb
 - spec/support/examples.rb
-- spec/wu_storm_spec.rb
+- spec/wukong-storm/bolt_driver_spec.rb
+- spec/wukong-storm/storm_invocation_spec.rb
+- spec/wukong-storm/storm_runner_spec.rb
+- spec/wukong-storm/wu-bolt_spec.rb
+- spec/wukong-storm/wu-storm_spec.rb
+- spec/wukong-storm_spec.rb
+- src/main/java/com/infochimps/wukong/storm/Builder.java
+- src/main/java/com/infochimps/wukong/storm/DataflowBuilder.java
+- src/main/java/com/infochimps/wukong/storm/SpoutBuilder.java
+- src/main/java/com/infochimps/wukong/storm/StateBuilder.java
+- src/main/java/com/infochimps/wukong/storm/TopologyBuilder.java
+- src/main/java/com/infochimps/wukong/storm/TopologySubmitter.java
 - wukong-storm.gemspec
 homepage: https://github.com/infochimps-labs/wukong-storm
 licenses:
@@ -67,7 +100,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3852445642211148218
+      hash: -2207093139671910492
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -76,15 +109,20 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3852445642211148218
+      hash: -2207093139671910492
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Storm processing for Ruby
 test_files:
 - spec/spec_helper.rb
 - spec/support/examples.rb
-- spec/wu_storm_spec.rb
+- spec/wukong-storm/bolt_driver_spec.rb
+- spec/wukong-storm/storm_invocation_spec.rb
+- spec/wukong-storm/storm_runner_spec.rb
+- spec/wukong-storm/wu-bolt_spec.rb
+- spec/wukong-storm/wu-storm_spec.rb
+- spec/wukong-storm_spec.rb
 has_rdoc: