wukong-storm 0.1.1 → 0.2.0

data/lib/wukong-storm/storm_runner.rb

@@ -0,0 +1,123 @@
+require_relative("storm_invocation")
+require 'kafka'
+
+module Wukong
+  module Storm
+
+    # Implements the runner for wu-storm.
+    class StormRunner < Wukong::Local::LocalRunner
+
+      # The default port Kafka is assumed to be running on.
+      DEFAULT_KAFKA_PORT = 9092
+
+      include Logging
+      include StormInvocation
+
+      usage "DATAFLOW|PROCESSOR"
+
+      description <<-EOF.gsub(/^ {8}/,'')
+wu-storm is a commandline tool for launching dynamically assembled
+parametrized Storm topologies that embed Wukong dataflows.  The
+overall "shape" of the launched topology is
+
+  spout -> wukong dataflow -> state
+
+The default spout and state will read and write to Kafka topics.
+Additional spouts and states are also available.
+
+Here's an example which launches `my_flow` as a Storm topology reading
+from the Kafka topic `raw` and writing to the Kafka topic `clean`:
+
+  $ wu-storm my_flow --input=raw --output=clean
+
+Here's an example which launches `my_flow` as a Storm topology reading
+from the S3 bucket `example-data` and the path `raw` and writing to
+the Kafka topic `clean`:
+
+  $ wu-storm my_flow --input=s3://example-data/raw --output=clean
+
+There are several options which apply to any topology like --name,
+--nimubs_host, --zookeeper_hosts, --ackers, --parallelism, &c.
+
+Some options only make sense when reading from Kafka like
+--kafka_batch, --kafka_partitions, or --input_parallelism.
+
+Other options only make sense when reading from S3 like --aws_key,
+--aws_secret, and --aws_region.
+
+Options like --from_beginning, --from_end, and --offset have general
+applicability but are interpeted differently based on the spout type.
+
+For a complete list of options try `wu storm --help`.
+      EOF
+
+      def kill_first?
+        settings[:rm]
+      end
+
+      def validate
+        begin
+          super()
+        rescue => e
+          raise e if dataflow
+          raise Error.new("Must provide a processor or dataflow to run, via either the --run option or as the first argument, or provide an explicit --bolt_command") unless settings[:bolt_command]
+        end
+        raise Error.new("An explicit --input URI is required to launch a dataflow")  if settings[:input].nil?  || settings[:input].empty?
+        raise Error.new("An explicit --output URI is required to launch a dataflow") if settings[:output].nil? || settings[:output].empty?
+
+        if kafka_input? || kafka_output?
+          raise Error.new("Must provide a list of comma-separated Kafka hosts")          if settings[:kafka_hosts].nil? || settings[:kafka_hosts].empty?
+        end
+        
+        if s3_input?
+          raise Error.new("Must provide an S3 bucket and path")                     if input_uri.path.nil?        || input_uri.path.empty?
+          raise Error.new("Must provide an AWS access key (settings[:aws_key])")    if settings[:aws_key].nil?    || settings[:aws_key].empty?
+          raise Error.new("Must provide an AWS secret key (settings[:aws_secret])") if settings[:aws_secret].nil? || settings[:aws_secret].empty?
+          raise Error.new("Invalid AWS region: <#{settings[:aws_region]}>") unless s3_endpoint
+        end
+        true
+      end
+
+      def run
+        log_topology_structure_and_settings
+        setup_run
+        if kill_first?
+          log.debug("Killing topology <#{topology_name}> and waiting <#{settings[:wait]}> seconds...")
+          execute_command(storm_kill_commandline)
+          sleep settings[:wait].to_i unless settings[:dry_run]
+        end
+        execute_command!(storm_launch_commandline)
+        raise Error.new("Failed to launch topology #{topology_name}!") unless settings[:dry_run] || $?.success?
+      end
+
+      protected
+
+      def log_topology_structure_and_settings
+        log.info("Using Zookeeper at <#{settings[:zookeeper_hosts]}>")
+        
+        log.info("Reading from Kafka <#{settings[:kafka_hosts]}/#{settings[:input]}>") if kafka_input?
+        log.info("Reading from filesystem at <#{settings[:input]}>")                   if blob_input?
+        log.info("Writing to Kafka <#{settings[:kafka_hosts]}/#{settings[:output]}>")  if kafka_output?
+
+        log.info("Dry run:") if settings[:dry_run]
+      end
+      
+      def setup_run
+        return unless kafka_input?
+        topic      = settings[:input]
+        host, port = kafka_host_and_port
+        log.info("Ensuring input topic <#{topic}> exists on Kafka broker <#{host}:#{port}>")
+        Kafka::Producer.new(host: host, port: port, topic: topic).push([]) unless settings[:dry_run]
+      end
+
+      def kafka_host_and_port
+        kafka_host = settings[:kafka_hosts].to_s.split(/ *, */).first
+        raise Error.new("Could not construct a Kafka host from <#{settings[:kafka_hosts]}>") unless kafka_host
+        parts = kafka_host.split(':')
+        raise Error.new("Badly formed Kafka host <#{kafka_host}>.  Must be in the format HOST[:PORT]") if parts.size > 2
+        [parts[0], (parts[1] || DEFAULT_KAFKA_PORT).to_i]
+      end
+      
+    end
+  end
+end

data/lib/wukong-storm/version.rb

@@ -1,5 +1,5 @@
 module Wukong
   module Storm
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/pom.xml

@@ -0,0 +1,111 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.infochimps.wukong</groupId>
+  <artifactId>wukong-storm</artifactId>
+  <packaging>jar</packaging>
+  <version>1.4.0-SNAPSHOT</version>
+  <name>wukong-storm</name>
+  <url>http://github.com/infochimps-labs/wukong-storm</url>
+
+  <parent>
+    <groupId>com.infochimps</groupId>
+    <artifactId>parent-pom</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+	<version>2.4</version>
+	<configuration>
+	  <descriptorRefs>
+	    <descriptorRef>jar-with-dependencies</descriptorRef>
+	  </descriptorRefs>
+	</configuration>
+	<executions>
+	  <execution>
+	    <id>dist-assembly</id>
+	    <phase>package</phase>
+	    <goals>
+	      <goal>single</goal>
+	    </goals>
+	  </execution>
+	</executions>
+      </plugin>
+    </plugins>
+    <sourceDirectory>src</sourceDirectory>
+  </build>
+  
+
+  <repositories>
+    <repository>
+      <id>Sonatype-public</id>
+      <name>SnakeYAML repository</name>
+      <url>http://oss.sonatype.org/content/groups/public/</url>
+    </repository>
+    <!-- Infochimps Repositories -->
+    <repository>
+      <id>infochimps.releases</id>
+      <name>Infochimps Internal Repository</name>
+      <url>https://s3.amazonaws.com/artifacts.chimpy.us/maven-s3p/releases</url>
+    </repository>
+    <repository>
+      <id>infochimps.snapshots</id>
+      <name>Infochimps Internal Repository</name>
+      <url>https://s3.amazonaws.com/artifacts.chimpy.us/maven-s3p/snapshots</url>
+      <snapshots>
+	<enabled>true</enabled>
+	<updatePolicy>always</updatePolicy>
+      </snapshots>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-core</artifactId>
+      <version>0.7.1-incubating</version>
+    </dependency>
+
+    <dependency>
+      <groupId>storm</groupId>
+      <artifactId>storm</artifactId>
+      <version>0.9.0</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>storm</groupId>
+      <artifactId>storm-kafka</artifactId>
+      <version>0.9.0-wip16a-scala292</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>2.1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-json-org</artifactId>
+      <version>2.1.2</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.infochimps</groupId>
+      <artifactId>storm-util</artifactId>
+      <version>1.7.0-SNAPSHOT</version>
+    </dependency>
+    
+  </dependencies>
+</project>

data/spec/spec_helper.rb

@@ -15,7 +15,19 @@ RSpec.configure do |config|
   end
 
   def storm_runner *args, &block
-    runner(Wukong::Storm::StormRunner, 'wu-storm', *args)
+    runner(Wukong::Storm::StormRunner, 'wu-storm', *args) do
+      stub(:execute_command)
+      instance_eval(&block) if block_given?
+    end
   end
+
+  def wu_storm *args
+    command('wu-storm', *args)
+  end
+  
+  def wu_bolt *args
+    command('wu-bolt', *args)
+  end
+  
   
 end

data/spec/wukong-storm/bolt_driver_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe Wukong::Storm::BoltDriver do
+
+  let(:settings) do
+    Configliere::Param.new.tap do |s|
+      Wukong::Storm.configure(s, 'wu-bolt')
+    end
+  end
+    
+  let(:driver)   { Wukong::Storm::BoltDriver.new(:bogus_event_machine_inserted_arg, :identity, settings) }
+
+  describe "setting up a dataflow" do
+    context "#post_init hook from EventMachine" do
+      after { driver.post_init }
+      it "should not sync $stdout" do
+        $stdout.should_not_receive(:sync)
+      end
+    end
+  end
+
+  describe "driving a dataflow" do
+    context "#receive_line hook from EventMachine" do
+      let(:line) { "hello" }
+      before { $stdout.stub(:write)     }
+      after  { driver.receive_line(line) }
+      it "passes the line to the #send_through_dataflow method" do
+        driver.should_receive(:send_through_dataflow).with(line)
+      end
+      it "calls the #write_output method" do
+        driver.should_receive(:write_output)
+      end
+      it "writes each output record" do
+        $stdout.should_receive(:write).with(line)
+      end
+      it "writes the batch delimiter" do
+        $stdout.should_receive(:write).with('X')
+      end
+      it "writes newlines after each output record and after the batch delimiter" do
+        $stdout.should_receive(:write).with("\n").exactly(2).times
+      end
+      
+    end
+  end
+
+end

data/spec/wukong-storm/storm_invocation_spec.rb

@@ -0,0 +1,204 @@
+require 'fileutils'
+require 'spec_helper'
+
+describe Wukong::Storm::StormInvocation do
+
+  before do
+    @producer = double("Kafka::Producer", push: true)
+    Kafka::Producer.stub(:new).and_return(@producer)
+  end
+  
+  context "without any options" do
+    subject { storm_runner('identity', '--input=foo', '--output=bar') }
+
+    its(:topology_name)                 { should == 'identity' }
+    its(:dataflow_name)                 { should == 'identity' }
+    
+    its(:kafka_input?)                  { should be_true }
+    its(:kafka_output?)                 { should be_true }
+
+    its(:storm_runner)             { should == 'storm' }
+    
+    its(:storm_launch_commandline) { should match(/jar .*storm.*.jar/)            }
+    its(:storm_launch_commandline) { should match(/com\.infochimps\..*TopologySubmitter/) }
+
+    its(:storm_kill_commandline)   { should match(/storm kill identity -w 300/) }
+  end
+
+  describe "Storm runner options" do
+    context "--storm_runner" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--storm_runner=/opt/bin/storm') }
+      its(:storm_runner) { should == '/opt/bin/storm' }
+    end
+    context "--storm_home" do
+      around do |example|
+        FileUtils.mkdir_p File.join(pwd, 'bin')
+        FileUtils.touch   File.join(pwd, 'bin', 'storm')
+        example.run
+        FileUtils.rm_r    File.join(pwd, 'bin')
+      end
+      let(:pwd){ File.expand_path('../..', __FILE__) }
+      subject { storm_runner('identity', '--input=foo', '--output=bar', "--storm_home=#{pwd}") }
+      its(:storm_runner) { should == File.join(pwd, 'bin/storm') }
+    end
+  end
+
+  describe "native Storm options" do
+    context "when setting --ackers" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--ackers=10') }
+      its(:storm_launch_commandline)   { should match(/topology.acker.executors=10/) }
+    end
+    context "when setting --wait" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--wait=1') }
+      its(:storm_kill_commandline)   { should match(/-w 1/) }
+    end
+  end
+
+  describe "services options" do
+    context "by default" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar') }
+      its(:storm_launch_commandline) { should match(/\wukong\.kafka\.hosts.*localhost/) }
+      its(:storm_launch_commandline) { should match(/\wukong\.zookeeper\.hosts.*localhost/) }
+    end
+    context "when setting --kafka_hosts" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--kafka_hosts=k1.example.com:9092,k2.example.com:9093') }
+      its(:storm_launch_commandline) { should match(/\wukong\.kafka\.hosts.*k1.example.com:9092,k2.example.com:9093/) }
+    end
+    context "when setting --zookeeper_hosts" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--zookeeper_hosts=z1.example.com:2181,z2.example.com:3181') }
+      its(:storm_launch_commandline) { should match(/\wukong\.zookeeper\.hosts.*z1.example.com:2181,z2.example.com:3181/) }
+    end
+  end
+
+  describe "topology options" do
+    context "by default" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar') }
+      its(:topology_name) { should == 'identity' }
+      its(:dataflow_name) { should == 'identity' }
+      its(:storm_launch_commandline) { should match(/\wukong\.topology.*identity/) }
+    end
+    context "when setting --name" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--name=myFlow') }
+      its(:topology_name) { should == 'myFlow' }
+      its(:dataflow_name) { should == 'identity' }
+      its(:storm_launch_commandline) { should match(/\wukong\.topology.*myFlow/) }
+    end
+  end
+
+  describe "spout options" do
+    context "when reading from Kafka" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar') }
+      its(:storm_launch_commandline) { should match(/\wukong\.input\.type.*kafka/)           }
+      its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.topic.*foo/)     }
+      its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.partitions.*1/)  }
+      its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.batch.*1048576/) }
+      its(:storm_launch_commandline) { should match(/\wukong\.input\.parallelism.*1/)        }
+      context "when setting --kafka_partitions" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--kafka_partitions=10') }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.partitions.*10/)  }
+      end
+      context "when setting --kafka_batch" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--kafka_batch=100') }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.batch.*10/) }
+      end
+      context "when setting --input_parallelism" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--input_parallelism=10') }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.parallelism.*10/) }
+      end
+      context "when setting --from_beginning" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--from_beginning') }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.offset.*-2/) }
+      end
+      context "when setting --from_end" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--from_end')    }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.offset.*-1/) }
+      end
+      context "when setting --offset" do
+        subject { storm_runner('identity', '--input=foo', '--output=bar', '--offset=1234')    }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.kafka\.offset.*1234/) }
+      end
+    end
+    context "when reading from a filesystem" do
+      context "when reading from a local filesystem" do
+        subject { storm_runner('identity', '--input=file:///foo/bar', '--output=baz')    }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.type.*blob/)             }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.type.*file/)       }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.path.*\/foo\/bar/) }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*RESUME/)    }
+        context "when setting --from_beginning" do
+          subject { storm_runner('identity', '--input=file:///foo/bar', '--output=baz', '--from_beginning')    }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*EARLIEST/) }
+        end
+        context "when setting --from_end" do
+          subject { storm_runner('identity', '--input=file:///foo/bar', '--output=baz', '--from_end')    }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*LATEST/) }
+        end
+        context "when setting --offset" do
+          subject { storm_runner('identity', '--input=file:///foo/bar', '--output=baz', '--offset=bing-1')    }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*EXPLICIT/) }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.marker.*bing-1/)  }
+        end
+      end
+      context "when reading from S3" do
+        subject { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret') }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.type.*blob/)               }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.type.*s3/)           }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.aws_key.*key/)       }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.aws_secret.*secret/) }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.s3_endpoint.*s3\.amazonaws\.com/) }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.s3_bucket.*foo/)     }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.path.*bar/)          }
+        its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*RESUME/)      }
+        context "when setting --from_beginning" do
+          subject { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret', '--from_beginning') }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*EARLIEST/) }
+        end
+        context "when setting --from_end" do
+          subject { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret', '--from_end') }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*LATEST/) }
+        end
+        context "when setting --offset" do
+          subject { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret', '--offset=bing-1') }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.start.*EXPLICIT/) }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.marker.*bing-1/)  }
+        end
+        context "when setting --aws_region" do
+          subject { storm_runner('identity', '--input=s3://foo/bar', '--output=baz', '--aws_key=key', '--aws_secret=secret', '--aws_region=us-west-1') }
+          its(:storm_launch_commandline) { should match(/\wukong\.input\.blob\.s3_endpoint.*s3-us-west-1\.amazonaws\.com/) }
+        end
+      end
+    end
+  end
+
+  describe "dataflow options" do
+    subject { storm_runner('identity', '--input=foo', '--output=bar') }
+    its(:storm_launch_commandline)   { should match(/wukong\.directory.*#{Dir.pwd}/) }
+    its(:storm_launch_commandline)   { should match(/wukong\.dataflow.*identity/) }
+    its(:storm_launch_commandline)   { should match(/wukong\.command.*wu-bolt.*identity/) }
+    its(:storm_launch_commandline)   { should match(/wukong\.parallelism.*1/) }
+    context "when setting --environment" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--environment=production') }
+      its(:storm_launch_commandline)   { should match(/wukong\.environment.*production/) }
+    end
+    context "when setting --command_prefix" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--command_prefix="bundle exec"') }
+      its(:storm_launch_commandline)   { should match(/wukong\.command.*bundle.*exec.*wu-bolt.*identity/) }
+    end
+    context "when setting --bolt_command" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--bolt_command="uniq -c"') }
+      its(:storm_launch_commandline)   { should match(/wukong\.command.*uniq.*-c/) }
+    end
+    context "when setting --parallelism" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar', '--parallelism=10') }
+      its(:storm_launch_commandline)   { should match(/wukong\.parallelism.*10/) }
+    end
+  end
+
+  describe "state options" do
+    context "when writing to Kafka" do
+      subject { storm_runner('identity', '--input=foo', '--output=bar') }
+      its(:storm_launch_commandline)   { should match(/wukong\.output\.kafka\.topic.*bar/) }
+    end
+  end
+  
+end