ul-wukong 4.1.0

data/lib/wukong/source/source_runner.rb

@@ -0,0 +1,38 @@
+require_relative('source_driver')
+module Wukong
+  module Source
+
+    # Implements the `wu-source` command.
+    class SourceRunner < Wukong::Local::LocalRunner
+
+      usage "PROCESSOR|DATAFLOW"
+
+      description <<-EOF.gsub(/^ {8}/,'')
+        wu-source is a tool for using Wukong processors as sources of
+        data in streams.
+
+        Run any Wukong processor as a source for data:
+
+          $ wu-source fake_log_data
+          205.4.75.208 - 3918471017 [27/Nov/2012:05:06:57 -0600] "GET /products/eget HTTP/1.0" 200 25600
+          63.181.105.15 - 3650805763 [27/Nov/2012:05:06:57 -0600] "GET /products/lacinia-nulla-vitae HTTP/1.0" 200 3790
+          227.190.78.101 - 39543891 [27/Nov/2012:05:06:58 -0600] "GET /products/odio-nulla-nulla-ipsum HTTP/1.0" 200 31718
+          ...
+
+        The fake_log_data processor will receive an event once every
+        second.  Each event will consist of a single string giving a
+        consecutive integer starting with '1' as the first event.
+      EOF
+      
+      include Logging
+
+      # The driver class used by `wu-source`.
+      #
+      # @return [Class]
+      def driver
+        SourceDriver
+      end
+
+    end
+  end
+end

data/lib/wukong/spec_helpers.rb

@@ -0,0 +1,74 @@
+require 'wukong'
+require_relative('spec_helpers/unit_tests')
+require_relative('spec_helpers/integration_tests')
+require_relative('spec_helpers/shared_examples')
+
+module Wukong
+  
+  # This module lets you use write processor specs at a high level.
+  # Require it in your <tt>spec_helper.rb</tt> file:
+  #
+  #   # in your spec/spec_helper.rb
+  #   require 'wukong/spec_helpers'
+  #   RSpec.configure do |config|
+  #     include Wukong::SpecHelpers
+  #   end
+  #   
+  # Processors in a Wukong spec will have a collection of
+  # <tt>given_*</tt> methods you can use to (lazily) feed them records
+  # without having to have to build your own driver to run the
+  # processors.
+  #
+  # To each <tt>given_*</tt> method corresponds an <tt>emit_*</tt>
+  # matcher which will actually run the processor on the given
+  # inputs and compare against expected results.  Here's an example,
+  # using a simple `tokenizer` processor.
+  #
+  #   subject { processor(:tokenizer) }
+  #
+  #   it "emits each word in a given string" do
+  #     given("It was the best of times, it was the worst of times.").should emit(12).records
+  #   end
+  #
+  #   # Give similar input and check against explicit expected
+  #   # output.
+  #   it "should ignore punctuation and capitalization" do
+  #     processor.given("You're crazy!").should emit("youre", "crazy")
+  #   end
+  #
+  #   # Pass the input but transform to JSON first (delimited and
+  #   # as_tsv also work).
+  #   it "should tokenize the 'text' attribute of a record if given JSON" do
+  #     processor.given("text" => "Will be cast to JSON").as_json.should emit("will", "be", "cast", "to", "json")
+  #   end
+  #
+  #   # Initialize the :tokenizer processor with arguments to test
+  #   # behavior under different conditions.
+  #   it "should output a single record when asked for JSON output" do
+  #     processor(:json => true).given("It was the best of times, it was the worst of times.").should emit(1).records
+  #   end
+  #
+  #   # Initialize processor with arguments and express that the
+  #   # expected output will be in JSON though given as an object.
+  #   it "should output all the tokens for its input record with its JSON output" do
+  #     processor(:json => true).given("You're crazy!").should emit("tokens" => ["youre", "crazy"]).as_json
+  #   end
+  #
+  #   # Initialize processor with arguments, and both input and
+  #   # output will be serialized/deserialized to/from JSON
+  #   # automatically.
+  #   it "can read and write pure JSON" do
+  #     processor(:json => true).given("text" => "You're crazy!").as_json.should emit("tokens" => ["youre", "crazy"]).as_json
+  #   end
+  #
+  #   # Use a processor outside the scope of the top-level :describe
+  #   # block.
+  #   it "has a friend which does the same thing" do
+  #     processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
+  #   end
+  module SpecHelpers
+    include UnitTests
+    include IntegrationTests
+  end
+end
+

data/lib/wukong/spec_helpers/integration_tests.rb

@@ -0,0 +1,150 @@
+require_relative('integration_tests/integration_test_runner')
+require_relative('integration_tests/integration_test_matchers')
+
+module Wukong
+  module SpecHelpers
+    
+    # This module defines methods that are helpful to use in
+    # integration tests which require reading files from the local
+    # repository.
+    #
+    # Integration tests will spawn new system processes with their own
+    # environments.  This module provides methods and hooks for
+    # customizing that environment.
+    module IntegrationTests
+
+      # The directory to add to the `RUBYLIB` environment variable for
+      # the spawned processes.
+      #
+      # If `args` are given, return a path within this directory.
+      #
+      # @param [Array<String>] args
+      # @return [String]
+      def lib_dir *args
+        root.join('lib', *args).to_s
+      end
+
+      # The directory to add to the `PATH` environment variable for
+      # the spawned processes.
+      #
+      # If `args` are given, return a path within this directory.
+      #
+      # @param [Array<String>] args
+      # @return [String]
+      def bin_dir *args
+        root.join('bin', *args).to_s
+      end
+
+      # The directory to use for examples for the spawned process.
+      #
+      # If `args` are given, return a path within this directory.
+      #
+      # @param [Array<String>] args
+      # @return [String]
+      def examples_dir *args
+        root.join('examples', *args).to_s
+      end
+
+      # A Hash of environment variables to use for the spawned
+      # process.
+      #
+      # By default, will put the IntegrationHelper#lib_dir on the
+      # `RUBYLIB` and the IntegrationHelper#bin_dir on the `PATH`.
+      #
+      # @return [Hash]
+      def integration_env
+        {
+          "PATH"    => [bin_dir.to_s, ENV["PATH"]].compact.join(':'),
+          "RUBYLIB" => [lib_dir.to_s, ENV["RUBYLIB"]].compact.join(':')
+        }
+      end
+
+      # The directory to spawn new processes in.
+      #
+      # @return [String]
+      def integration_cwd
+        root.to_s
+      end
+
+      # Checks that each `expectation` appears in the STDOUT of the
+      # command.  Order is irrelevant and each `expectation` can be
+      # either a String to check for inclusion or a Regexp to match
+      # with.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def have_stdout *expectations
+        StdoutMatcher.new(*expectations)
+      end
+
+      # Checks that each `expectation` appears in the STDERR of the
+      # command.  Order is irrelevant and each `expectation` can be
+      # either a String to check for inclusion or a Regexp to match
+      # with.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def have_stderr *expectations
+        StderrMatcher.new(*expectations)
+      end
+
+      # Checks that the command exits with the given `code`.
+      #
+      # @param [Integer] code
+      def exit_with code
+        ExitCodeMatcher.new(code)
+      end
+      
+      # Spawn a command and capture its STDOUT, STDERR, and exit code.
+      #
+      # The `args` will be joined together into a command line.
+      #
+      # It is expected that you will use the matchers defined in
+      # IntegrationMatchers in your integration tests:
+      #
+      # @example Check output of 'ls' includes a string 'foo.txt'
+      #   it "lists files" do
+      #     command('ls').should have_output('foo.txt')
+      #   end
+      #
+      # @example More complicated
+      #   context "long format" do
+      #     it "lists files with timestamps" do
+      #       command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
+      #     end
+      #   end
+      #
+      # @param [Array<String>] args
+      #
+      # @overload command(*args, options={})
+      #   If the last element of `args` is a Hash it will be used for
+      #   options.
+      #
+      #   The :env option specifies the command line environment to
+      #   use for the command.  By default this will be the value of
+      #   the Ruby process's own `ENV` variable.  If running in a
+      #   context in which the `integration_env` method is defined,
+      #   its return value will be merged on top of `ENV`.  An
+      #   explicitly provided :env option will again be merged on top.
+      #
+      #   The :cwd option specifies the working directory to start in.
+      #   It defaults to the value of <tt>Dir.pwd</tt>
+      #
+      #   @param [Array<String>] args
+      #   @param [Hash] options
+      #   @option options [Hash] env the shell environment to spawn the command with
+      #   @option options [Hash] cwd the directory to execute the command in
+      def command *args
+        a = args.flatten.compact
+        options = (a.last.is_a?(Hash) ? a.pop : {})
+
+        env = ENV.to_hash.dup
+        env.merge!(integration_env) if respond_to?(:integration_env)
+        env.merge!(options[:env] || {})
+
+        cwd   = options[:cwd]
+        cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
+
+        IntegrationTestRunner.new(a, cwd: cwd, env: env)
+      end
+    end
+  end
+end

data/lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb

@@ -0,0 +1,207 @@
+module Wukong
+  module SpecHelpers
+    
+    # A class for running commands and capturing their STDOUT, STDERR,
+    # and exit code.  This class is designed to work with the matchers
+    # defined in IntegrationTestMatchers.
+    class IntegrationTestMatcher
+
+      # The runner used to run the actual commands.
+      attr_accessor :runner
+
+      # An array of expectations about the output of the runner.
+      attr_accessor :expectations
+
+      # The expectation which caused failure.
+      attr_accessor :failed_expectation
+
+      # Return whether or not the given command's output matches
+      # expectations.
+      #
+      # If an expectation failes to match, the `failed_expectation`
+      # attribute will be set accordingly.
+      #
+      # @param [IntegrationTestRunner] runner
+      # @return [true, false]
+      def matches?(runner)
+        self.runner = runner
+        runner.run!
+        expectations.each do |expectation|
+          unless output.send(match_function(expectation), expectation)
+            self.failed_expectation = expectation
+            return false
+          end
+        end
+        true
+      end
+
+      # Create a matcher on the given expectations.  Each expectation
+      # can be either a String or a Regexp.  Strings will be tested
+      # for inclusion in the output, Regexps will be tested for a
+      # match against the output.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def initialize *expectations
+        self.expectations = expectations
+      end
+
+      # :nodoc:
+      def failure_message
+        "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n  #{failed_expectation}#{formatted_error_output}"
+      end
+
+      # :nodoc:
+      def negative_failure_message
+        "Expected #{output_description} of #{runner.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
+      end
+
+      # :nodoc:
+      def formatted_output
+        output.split("\n").map { |line| '  ' + line }.join("\n")
+      end
+
+      # :nodoc:
+      def formatted_error_output
+        output_description.to_s =~ /stderr/ ? "\n\nSTDOUT was\n\n#{runner.stdout}" : "\n\nSTDERR was\n\n#{runner.stderr}"
+      end
+
+      # :nodoc:
+      def formatted_command
+        "  $ #{runner.cmd}"
+      end
+
+      # :nodoc:
+      def formatted_env
+        ['  {'].tap do |lines|
+          runner.env.each_pair do |key, value|
+            if key =~ /^(BUNDLE_GEMFILE|PATH|RUBYLIB)$/
+              lines << "    #{key} => #{value},"
+            end
+          end
+          lines << '  }'
+        end.join("\n")
+      end
+
+      # :nodoc:
+      def match_function expectation
+        expectation.is_a?(Regexp) ? :match : :include?
+      end
+
+      # :nodoc:
+      def match_type
+        failed_expectation.is_a?(Regexp) ? 'match' : 'include'
+      end
+
+    end
+
+    # A matcher for the STDOUT of a command.
+    class StdoutMatcher < IntegrationTestMatcher
+
+      # Picks the STDOUT of the command.
+      def output
+        runner.stdout
+      end
+
+      # :nodoc:
+      def output_description
+        "STDOUT"
+      end
+
+      def description
+        "have the correct #{output_description}"
+      end
+      
+    end
+
+    # A matcher for the STDOUT of a command.
+    class StderrMatcher < IntegrationTestMatcher
+
+      # Picks the STDOUT of the command.
+      def output
+        runner.stderr
+      end
+
+      # :nodoc:
+      def output_description
+        "STDERR"
+      end
+
+      def description
+        "print an appropriate error message on #{output_description}"
+      end
+    end
+
+    # A matcher for the exit code of a command.
+    class ExitCodeMatcher < IntegrationTestMatcher
+
+      # Initialize this matcher with the given `code`.
+      #
+      # If `code` is the symbol <tt>:non_zero</tt> then the
+      # expectation will be any non-zero exit code.
+      #
+      # @param [Integer,Symbol] code
+      def initialize code
+        if code ==  :non_zero
+          @expected_code = :non_zero
+        else
+          @expected_code = code.to_i
+        end
+      end
+
+      # Return whether or not the given command's exit code matches
+      # the expectation.
+      #
+      # @param [IntegrationTestRunner] runner
+      # @return [true, false]
+      def matches?(runner)
+        self.runner = runner
+        runner.run!
+        if non_zero_exit_code?
+          @failed = true if runner.exit_code == 0
+        else
+          @failed = true if runner.exit_code != expected_exit_code
+        end
+        @failed ? false : true
+      end
+
+      # :nodoc:
+      def failure_message
+        "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nexpecting #{expected_exit_code_description}  Got #{runner.exit_code} instead.#{formatted_error_output}"
+      end
+
+      # :nodoc:
+      def negative_failure_message
+        "From within #{runner.cwd} ran\n\n#{formatted_env}\n#{formatted_command}\n\nNOT expecting #{expected_exit_code_description}.#{formatted_error_output}"
+      end
+
+      # :nodoc:
+      def non_zero_exit_code?
+        @expected_code == :non_zero
+      end
+
+      # :nodoc:
+      def expected_exit_code
+        (@expected_code || 0).to_i
+      end
+
+      # :nodoc:
+      def expected_exit_code_description
+        if non_zero_exit_code?
+          "a non-zero exit code"
+        else
+          "an exit code of #{expected_exit_code}"
+        end
+      end
+
+      # :nodoc:
+      def description
+        "exit with #{expected_exit_code_description}"
+      end
+
+      # :nodoc:
+      def output_description
+        "STDOUT"
+      end
+    end
+  end
+end