wukong-storm 0.1.1 → 0.2.0

data/spec/wukong-storm/storm_runner_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe Wukong::Storm::StormRunner do
+
+  before do
+    @producer = double("Kafka::Producer", push: true)
+    Kafka::Producer.stub(:new).and_return(@producer)
+  end
+
+  describe "validating a topology about to be launched" do
+    it "raises an error without a dataflow (or an explicit --bolt_command) to run" do
+      expect { storm_runner('--input=foo', '--output=bar') }.to raise_error(Wukong::Error, /processor.*dataflow.*run/i)
+    end
+
+    it "raises an error on a non-existing dataflow" do
+      expect { storm_runner('definitelyNotGonnaBeThere', '--input=foo', '--output=bar') }.to raise_error(Wukong::Error, /definitelyNotGonnaBeThere/)
+    end
+
+    context "reading and writing from Kafka" do
+      it "raises an error without an --input topic" do
+        expect { storm_runner('identity', '--output=bar') }.to raise_error(Wukong::Error, /input.*required/i)
+      end
+    
+      it "raises an error without an --output topic" do
+        expect { storm_runner('identity', '--input=foo') }.to raise_error(Wukong::Error, /output.*required/i)
+      end
+
+      it "raises an error when --kafka_hosts is empty or missing" do
+        expect { storm_runner('identity', '--input=foo', '--output=bar', '--kafka_hosts=') }.to raise_error(Wukong::Error, /kafka.*host/i)
+      end
+    end
+
+    context "reading from S3 and writing to Kafka" do
+      it "raises an error without a path" do
+        expect { storm_runner('identity', '--input=s3://foo', '--output=baz', '--aws_key=key', '--aws_secret=secret') }.to raise_error(Wukong::Error, /s3.*path/i)
+      end
+      it "raises an error without an AWS access key" do
+        expect { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_secret=secret') }.to raise_error(Wukong::Error, /aws.*key/i)
+      end
+      
+      it "raises an error without an AWS secret key" do
+        expect { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key') }.to raise_error(Wukong::Error, /aws.*secret/i)
+      end
+
+      it "raises an error on an invalid AWS region" do
+        expect { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret', '--aws_region=us-east-7') }.to raise_error(Wukong::Error, /aws.*region/i)
+      end
+      
+    end
+  end
+
+  describe "setting up for a topology about to be launched" do
+    context "when reading from Kafka" do
+      it "ensures the Kafka input topic exists" do
+        Kafka::Producer.should_receive(:new).with(host: 'localhost', port: 9092, topic: 'foo')
+        @producer.should_receive(:push).with([])
+        storm_runner('identity', '--input=foo', '--output=bar')
+      end
+    end
+  end
+
+  describe "killing a running topology before launching a new one" do
+    it "will not try to kill a previously running topology first" do
+      storm_runner('identity', '--input=foo', '--output=bar', '--wait=1') do
+        should_not_receive(:execute_command).with(/storm.*kill/)
+      end
+    end
+
+    it "will try to kill a previously running topology if asked" do
+      storm_runner('identity', '--rm', '--input=foo', '--output=bar', '--wait=1') do
+        should_receive(:execute_command).with(/storm.*kill/)
+      end
+    end
+  end
+end
+

data/spec/{wu_storm_spec.rb → wukong-storm/wu-bolt_spec.rb}

@@ -8,47 +8,47 @@ Wu.processor(:test) do
   
 end
 
-describe 'wu-storm' do
-  let(:examples) { File.expand_path('../support/examples.rb', __FILE__)   }
+describe 'wu-bolt' do
+  let(:examples) { File.expand_path('../../support/examples.rb', __FILE__)   }
 
   context 'without any arguments' do    
-    subject      { command 'wu-storm'                                     }
+    subject      { wu_bolt                                                }
     it           { should exit_with(:non_zero)                            }
-    it           { should have_stderr(/provide a processor.*to run/)      }
+    it           { should have_stderr(/provide a.*dataflow.*run/)         }
   end
 
   context 'with a simple processor' do
     let(:input)  { 'one event'                                            } 
-    subject      { command('wu-storm', examples, '--run=simple') < input  }
+    subject      { wu_bolt(examples, '--run=simple') < input  }
     it           { should exit_with(0)                                    }
-    it           { should have_stdout("one event\n---\n")                 }
+    it           { should have_stdout("one event\nX\n")                 }
   end
 
   context 'with a skipped processor' do
     let(:input)  { 'never see this'                                       }
-    subject      { command('wu-storm', examples, '--run=skipped') < input }
+    subject      { wu_bolt(examples, '--run=skipped') < input }
     it           { should exit_with(0)                                    }
-    it           { should have_stdout("---\n")                            }
+    it           { should have_stdout("X\n")                            }
   end
 
   context 'with a duplicating processor' do
     let(:input)  { 'foo'                                                  }
-    subject      { command('wu-storm', examples, '--run=multi') < input   }
+    subject      { wu_bolt(examples, '--run=multi') < input   }
     it           { should exit_with(0)                                    }
-    it           { should have_stdout("foo\nfoo\nfoo\n---\n")             }
+    it           { should have_stdout("foo\nfoo\nfoo\nX\n")             }
   end
 
   context 'with a flow' do
     let(:input)  { '{"foo":"bar"}'                                        }
-    subject      { command('wu-storm', examples, '--run=flow') < input    }
+    subject      { wu_bolt(examples, '--run=flow') < input    }
     it           { should exit_with(0)                                    }
-    it           { should have_stdout("I raised the bar\n---\n")          }
+    it           { should have_stdout("I raised the bar\nX\n")          }
   end  
 
   context 'with multiple arguments' do
     let(:input)  { "foo\nbar\nbaz"                                        }
-    subject      { command('wu-storm', examples, '--run=simple') < input  }
+    subject      { wu_bolt(examples, '--run=simple') < input  }
     it           { should exit_with(0)                                    }
-    it           { should have_stdout("foo\n---\nbar\n---\nbaz\n---\n")   }
+    it           { should have_stdout("foo\nX\nbar\nX\nbaz\nX\n")   }
   end  
 end

data/spec/wukong-storm/wu-storm_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe 'wu-storm' do
+  context "without any arguments" do
+    let(:subject) { command('wu-storm') }
+    it {should exit_with(:non_zero) }
+    it "displays help on STDERR" do
+      should have_stderr(/processor.*dataflow.*run.*bolt_command/i)
+    end
+  end
+
+  context "in --dry_run mode" do
+    let(:subject) { command('wu-storm', 'identity', "--input=foo", "--output=foo", "--dry_run") }
+    it { should exit_with(0) }
+    it { should have_stdout(/storm.*jar/, /TopologySubmitter/, /wu-bolt.*identity/) }
+  end
+end

data/spec/wukong-storm_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe Wukong::Storm do
+  it_behaves_like 'a plugin'
+end

data/src/main/java/com/infochimps/wukong/storm/Builder.java

@@ -0,0 +1,53 @@
+package com.infochimps.wukong.storm;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+
+public class Builder {
+
+    static Logger LOG = Logger.getLogger(StateBuilder.class);    
+
+    public Builder() {
+    }
+
+    public Boolean valid() {
+	return true;
+    }
+
+    public void logInfo() {
+    }
+    
+    static public String usage() {
+	return "";
+    }
+    
+    public static String ZOOKEEPER_HOSTS		= "wukong.zookeeper.hosts";
+    public static String DEFAULT_ZOOKEEPER_HOSTS	= "localhost";
+    public String zookeeperHosts() {
+	return prop(ZOOKEEPER_HOSTS, DEFAULT_ZOOKEEPER_HOSTS);
+    }
+
+    public static String KAFKA_HOSTS			= "wukong.kafka.hosts";
+    public static String DEFAULT_KAFKA_HOSTS		= "localhost";
+    public List<String> kafkaHosts() {
+	ArrayList<String> kh = new ArrayList();
+	for (String host : prop(KAFKA_HOSTS, DEFAULT_KAFKA_HOSTS).split(",")) {
+	    kh.add(host);
+	}
+	return kh;
+    }
+    
+    public String prop(String key, String defaultValue) {
+	if (System.getProperty(key) == null) {
+	    System.setProperty(key, defaultValue);
+	}
+	return prop(key);
+    }
+
+    public String prop(String key) {
+	return System.getProperty(key);
+    }
+    
+} 

data/src/main/java/com/infochimps/wukong/storm/DataflowBuilder.java

@@ -0,0 +1,74 @@
+package com.infochimps.wukong.storm;
+
+import org.apache.log4j.Logger;
+
+import com.infochimps.storm.wukong.WuFunction;
+
+public class DataflowBuilder extends Builder {
+
+    static Logger LOG = Logger.getLogger(DataflowBuilder.class);
+
+    private SpoutBuilder spoutBuilder;
+
+    public DataflowBuilder(SpoutBuilder spoutBuilder) {
+	this.spoutBuilder = spoutBuilder;
+    }
+
+    @Override
+    public Boolean valid() {
+	if (dataflowName() == null) {
+	    LOG.error("Must set a dataflow name using the " + DATAFLOW_NAME + " property");
+	    return false;
+	};
+	return true;
+    }
+
+    @Override
+    public void logInfo() {
+	LOG.info("DATAFLOW: Launching Wukong dataflow <" + dataflowName() + "> with parallelism " + dataflowParallelism() + " in environment <" + dataflowEnv() + ">" );
+    }
+
+    public static String usage() {
+	String s = "DATAFLOW OPTIONS\n"
+	    + "\n"
+	    + "The following options can be applied to the dataflow connecting the spout to the state:\n"
+	    + "\n"
+	    + "  " + String.format("%10s", DATAFLOW_NAME) + "  Name of the Wukong dataflow to launch (Required)\n"
+	    + "  " + String.format("%10s", DATAFLOW_ENV) + "  Wukong environment (Default: " + DEFAULT_DATAFLOW_ENV + ")\n"
+	    + "  " + String.format("%10s", BOLT_COMMAND) + "  The command-line to execute within a Storm bolt (Required)\n"
+	    + "  " + String.format("%10s", DATAFLOW_DIRECTORY) + "  The directory within which to execute the command-line (Default: " + DEFAULT_DATAFLOW_DIRECTORY + ")\n"
+	    + "  " + String.format("%10s", DATAFLOW_PARALLELISM) + "  Parallelism hint for Wukong dataflow Trident function (Default: same as --input_parallelism)\n";
+	return s;
+    }
+    
+    public WuFunction dataflow() {
+	return new WuFunction(dataflowName(), subprocessDirectory(), dataflowEnv());
+    }
+
+    public static String DATAFLOW_DIRECTORY             = "wukong.directory";
+    public static String DEFAULT_DATAFLOW_DIRECTORY     = System.getProperty("user.dir");
+    public String subprocessDirectory() {
+	return prop(DATAFLOW_DIRECTORY, DEFAULT_DATAFLOW_DIRECTORY);
+    }
+
+    public static String DATAFLOW_NAME			= "wukong.dataflow";
+    public String dataflowName() {
+	return prop(DATAFLOW_NAME);
+    }
+
+    // This is actually used directly by WuFunction but it's listed
+    // here for completeness since it is set by the Ruby code.
+    public static String BOLT_COMMAND                   = "wukong.command";
+
+    public static String DATAFLOW_ENV			= "wukong.environment";
+    public static String DEFAULT_DATAFLOW_ENV	        = "development";
+    public String dataflowEnv() {
+	return prop(DATAFLOW_ENV, DEFAULT_DATAFLOW_ENV);
+    }
+    
+    public static String DATAFLOW_PARALLELISM		= "wukong.parallelism";
+    public int dataflowParallelism() {
+	return Integer.parseInt(prop(DATAFLOW_PARALLELISM, Integer.toString(spoutBuilder.inputParallelism())));
+    }
+    
+}

data/src/main/java/com/infochimps/wukong/storm/SpoutBuilder.java

@@ -0,0 +1,237 @@
+package com.infochimps.wukong.storm;
+
+import java.lang.IllegalArgumentException;
+
+import org.apache.log4j.Logger;
+
+import backtype.storm.spout.SchemeAsMultiScheme;
+
+import storm.trident.spout.IOpaquePartitionedTridentSpout;
+
+import storm.kafka.KafkaConfig;
+import storm.kafka.StringScheme;
+import storm.kafka.trident.OpaqueTridentKafkaSpout;
+import storm.kafka.trident.TridentKafkaConfig;
+
+import com.infochimps.storm.trident.spout.OpaqueTransactionalBlobSpout;
+import com.infochimps.storm.trident.spout.StartPolicy;
+import com.infochimps.storm.trident.spout.WukongRecordizer;
+import com.infochimps.storm.trident.spout.IBlobStore;
+import com.infochimps.storm.trident.spout.S3BlobStore;
+import com.infochimps.storm.trident.spout.FileBlobStore;
+
+public class SpoutBuilder extends Builder {
+
+    static Logger LOG = Logger.getLogger(SpoutBuilder.class);
+
+    @Override
+    public Boolean valid() {
+	if (spoutType().equals(KAFKA_SPOUT_TYPE)) {
+	    if (kafkaInputTopic() == null) {
+		LOG.error("Must set an input topic name using the " + KAFKA_INPUT_TOPIC + " property when using a Kafka spout");
+		return false;
+	    };    
+	}
+	if (spoutType().equals(BLOB_SPOUT_TYPE)) {
+	    if (blobStorePath() == null) {
+		LOG.error("Must set a path using the " + BLOB_STORE_PATH + " property when using a blob store spout");
+		return false;
+	    };
+	    if (blobStoreType().equals(S3_BLOB_TYPE)) {
+		if (s3Bucket() == null) {
+		    LOG.error("Must set an S3 bucket using the " + S3_BUCKET + " property when using the S3 spout");
+		    return false;
+		};
+		if (awsKey() == null) {
+		    LOG.error("Must set an AWS access key using the " + AWS_KEY + " property when using the S3 spout");
+		    return false;
+		};
+		if (awsSecret() == null) {
+		    LOG.error("Must set an AWS secret key using the " + AWS_SECRET + " property when using the S3 spout");
+		    return false;
+		};
+	    }
+	}
+	return true;
+    }
+
+    @Override
+    public void logInfo() {
+	if (spoutType().equals(BLOB_SPOUT_TYPE)) {
+	    if (blobStoreType().equals(S3_BLOB_TYPE)) {
+		LOG.info("SPOUT: Reading from S3 bucket s3://" + s3Bucket() + " at path /" + blobStorePath() + ", using AWS key " + awsKey());
+	    } else {
+		LOG.info("SPOUT: Reading from local file file:///" + blobStorePath());
+	    }
+	} else {
+	    LOG.info("SPOUT: Reading from offset " + kafkaInputOffset() + " of Kafka topic <" + kafkaInputTopic() + "> in batches of " + kafkaInputBatchSize() + " with parallelism " + inputParallelism());
+	}
+    }
+
+    public static String usage() {
+	String s = "SPOUT OPTIONS\n"
+	    + "\n"
+	    + "Choose the spout with he following properties.  Each spout has its own further\n"
+	    + "configuration\n"
+	    + "\n"
+            + "  Kafka Spout -- " + SPOUT_TYPE + "=" + KAFKA_SPOUT_TYPE + "\n"
+	    + "  BlobStore Spout -- " + SPOUT_TYPE + "=" + BLOB_TYPE + "\n"
+	    + "    Filesystem Spout -- " + BLOB_SPOUT_TYPE + "=" + FILE_BLOB_TYPE + "\n"
+	    + "    S3 Spout -- " + BLOB_SPOUT_TYPE + "=" + S3_BLOB_TYPE + "\n"
+	    + "\n"
+	    + "The following options apply for the Kafka spout (" + SPOUT_TYPE + "=" + KAFKA_SPOUT_TYPE + "):\n"
+	    + "\n"
+	    + "  " + String.format("%10s", INPUT_PARALLELISM) + "	Parallelism hint for the spout (Default: " + DEFAULT_INPUT_PARALLELISM + ")\n"
+	    + "  " + String.format("%10s", KAFKA_INPUT_TOPIC) + "  Name of the Kafka topic to read input from"
+	    + "  " + String.format("%10s", KAFKA_INPUT_OFFSET) + "  Offset from which to start consuming from the input topic, one of: -1 = 'end', -2 = 'beginning', or an explicit byte offset.  (Default: resume if possible, else '1')\n"
+	    + "  " + String.format("%10s", KAFKA_INPUT_PARTITIONS) + "  Number of Storm partitions to use.  Should match the number of partitions on the input topic. (Default: " + DEFAULT_KAFKA_INPUT_PARTITIONS + ")\n"
+	    + "  " + String.format("%10s", KAFKA_INPUT_BATCH) + "  Batch size to fetch from Kafka (Default: " + DEFAULT_KAFKA_INPUT_BATCH + ")\n"
+	    + "\n"
+	    + "The following options apply for all BlobStore spouts (" + SPOUT_TYPE + "=" + BLOB_TYPE + "):\n"
+	    + "\n"
+	    + "  " + String.format("%10s", BLOB_STORE_PATH) + "  Directory to read from (Required)\n"
+	    + "  " + String.format("%10s", BLOB_START) + "  Starting policy, one of: EARLIEST, LATEST, EXPLICIT, or RESUME.  (Default: 'RESUME' if possible, else 'LATEST')\n"
+	    + "  " + String.format("%10s", BLOB_MARKER) + "  Required name of marker for an EXPLICIT starting policy\n"
+	    + "\n"
+	    + "The following options apply for the S3 spout (" + BLOB_SPOUT_TYPE + "=" + S3_BLOB_TYPE + "):\n"
+	    + "\n"
+	    + "  " + String.format("%10s", S3_BUCKET) + "  S3 bucket (Required)\n"
+	    + "  " + String.format("%10s", AWS_KEY) + "  AWS access key (Required)\n"
+	    + "  " + String.format("%10s", AWS_SECRET) + "  AWS secret key (Required)\n";
+	return s;
+    }
+    
+    public IOpaquePartitionedTridentSpout spout() {
+	if (spoutType().equals(BLOB_SPOUT_TYPE)) {
+	    return new OpaqueTransactionalBlobSpout(blobStore(), new WukongRecordizer(), blobStart(), blobMarker());
+	} else {
+	    return new OpaqueTridentKafkaSpout(kafkaSpoutConfig());
+	}
+    }
+
+    private IBlobStore blobStore() {
+	if (blobStoreType().equals(S3_BLOB_TYPE)) {
+	    return new S3BlobStore(blobStorePath(), s3Bucket(), s3Endpoint(), awsKey(), awsSecret());
+	} else {
+	    return new FileBlobStore(blobStorePath());
+	}
+    }
+
+    private TridentKafkaConfig kafkaSpoutConfig() {
+	TridentKafkaConfig kafkaConfig = new TridentKafkaConfig(KafkaConfig.StaticHosts.fromHostString(kafkaHosts(), kafkaInputPartitions()), kafkaInputTopic());
+	kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
+	kafkaConfig.fetchSizeBytes = kafkaInputBatchSize();
+	kafkaConfig.forceStartOffsetTime(kafkaInputOffset());
+	return kafkaConfig;
+    }
+    
+    public static String INPUT_PARALLELISM		= "wukong.input.parallelism";
+    public static String DEFAULT_INPUT_PARALLELISM	= "1";
+    public int inputParallelism() {
+	return Integer.parseInt(prop(INPUT_PARALLELISM, DEFAULT_INPUT_PARALLELISM));
+    }
+
+    public static String SPOUT_TYPE                     = "wukong.input.type";
+    public static String KAFKA_SPOUT_TYPE               = "kafka";
+    public static String BLOB_SPOUT_TYPE                = "blob";
+    public String spoutType() {
+	if ((prop(SPOUT_TYPE) != null) && prop(SPOUT_TYPE).equals(BLOB_SPOUT_TYPE)) {
+	    return BLOB_SPOUT_TYPE;
+	} else {
+	    return KAFKA_SPOUT_TYPE;
+	}
+    }
+
+    public Boolean isBlobSpout() {
+	return spoutType().equals(BLOB_SPOUT_TYPE);
+    }
+
+    public Boolean isKafkaSpout() {
+	return spoutType().equals(KAFKA_SPOUT_TYPE);
+    }
+
+    public static String BLOB_STORE_PATH                = "wukong.input.blob.path";
+    public String blobStorePath() {
+	return prop(BLOB_STORE_PATH);
+    }
+
+    public static String BLOB_TYPE                      = "wukong.input.blob.type";
+    public static String FILE_BLOB_TYPE                 = "file";
+    public static String S3_BLOB_TYPE                   = "s3";
+    public String blobStoreType() {
+	if ((prop(BLOB_TYPE) != null) && prop(BLOB_TYPE).equals(S3_BLOB_TYPE)){
+	    return S3_BLOB_TYPE;
+	} else {
+	    return FILE_BLOB_TYPE;
+	}
+    }
+
+    public Boolean isS3Spout() {
+	return (isBlobSpout() && blobStoreType().equals(S3_BLOB_TYPE));
+    }
+
+    public Boolean isFileSpout() {
+	return (isBlobSpout() && blobStoreType().equals(FILE_BLOB_TYPE));
+    }
+	    
+    public static String BLOB_START         = "wukong.input.blob.start";
+    public static String DEFAULT_BLOB_START = "RESUME";
+    public StartPolicy blobStart() {
+	try {
+	    return StartPolicy.valueOf(prop(BLOB_START, DEFAULT_BLOB_START));
+	} catch (IllegalArgumentException e) {
+	    return StartPolicy.RESUME;
+	}
+    }
+
+    public static String BLOB_MARKER = "wukong.input.blob.marker";
+    public String blobMarker() {
+	return prop(BLOB_MARKER);
+    }
+
+    public static String S3_BUCKET                      = "wukong.input.blob.s3_bucket";
+    public String s3Bucket() {
+	return prop(S3_BUCKET);
+    }
+
+    public static String S3_ENDPOINT                    = "wukong.input.blob.s3_endpoint";
+    public static String DEFAULT_S3_ENDPOINT            = "s3.amazonaws.com";
+    public String s3Endpoint() {
+	return prop(S3_ENDPOINT, DEFAULT_S3_ENDPOINT);
+    }
+    
+    public static String AWS_KEY                        = "wukong.input.blob.aws_key";
+    public String awsKey() {
+	return prop(AWS_KEY);
+    }
+    
+    public static String AWS_SECRET                     = "wukong.input.blob.aws_secret";
+    public String awsSecret() {
+	return prop(AWS_SECRET);
+    }
+
+    public static String KAFKA_INPUT_TOPIC        	= "wukong.input.kafka.topic";
+    public String kafkaInputTopic() {
+	return prop(KAFKA_INPUT_TOPIC);
+    }
+    
+    public static String KAFKA_INPUT_OFFSET		= "wukong.input.kafka.offset";
+    public static String DEFAULT_KAFKA_INPUT_OFFSET     = "-1";
+    public Integer kafkaInputOffset() {
+	return Integer.parseInt(prop(KAFKA_INPUT_OFFSET, DEFAULT_KAFKA_INPUT_OFFSET));
+    }
+    
+    public static String KAFKA_INPUT_PARTITIONS		= "wukong.input.kafka.partitions";
+    public static String DEFAULT_KAFKA_INPUT_PARTITIONS	= "1";
+    public int kafkaInputPartitions() {
+	return Integer.parseInt(prop(KAFKA_INPUT_PARTITIONS, DEFAULT_KAFKA_INPUT_PARTITIONS));
+    }
+    
+    public static String KAFKA_INPUT_BATCH			= "wukong.input.kafka.batch";
+    public static String DEFAULT_KAFKA_INPUT_BATCH		= "1048576";
+    public int kafkaInputBatchSize() {
+	return Integer.parseInt(prop(KAFKA_INPUT_BATCH, DEFAULT_KAFKA_INPUT_BATCH));
+    }
+    
+    
+}