wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/spec_helpers.rb

@@ -0,0 +1,81 @@
+require 'wukong'
+require 'wukong/boot'
+require_relative('spec_helpers/processor_helpers')
+require_relative('spec_helpers/processor_methods')
+require_relative('spec_helpers/spec_driver_matchers')
+require_relative('spec_helpers/integration_driver')
+require_relative('spec_helpers/integration_driver_matchers')
+require_relative('spec_helpers/shared_examples')
+
+module Wukong
+  
+  # This module lets you use write processor specs at a high level.
+  # Require it in your <tt>spec_helper.rb</tt> file:
+  #
+  #   # in your spec/spec_helper.rb
+  #   require 'wukong/spec_helpers'
+  #   RSpec.configure do |config|
+  #     include Wukong::SpecHelpers
+  #   end
+  #   
+  # Processors in a Wukong spec will have a collection of
+  # <tt>given_*</tt> methods you can use to (lazily) feed them records
+  # without having to have to build your own driver to run the
+  # processors.
+  #
+  # To each <tt>given_*</tt> method corresponds an <tt>emit_*</tt>
+  # matcher which will actually run the processor on the given
+  # inputs and compare against expected results.  Here's an example,
+  # using a simple `tokenizer` processor.
+  #
+  #   subject { processor(:tokenizer) }
+  #
+  #   it "emits each word in a given string" do
+  #     given("It was the best of times, it was the worst of times.").should emit(12).records
+  #   end
+  #
+  #   # Give similar input and check against explicit expected
+  #   # output.
+  #   it "should ignore punctuation and capitalization" do
+  #     processor.given("You're crazy!").should emit("youre", "crazy")
+  #   end
+  #
+  #   # Pass the input but transform to JSON first (delimited and
+  #   # as_tsv also work).
+  #   it "should tokenize the 'text' attribute of a record if given JSON" do
+  #     processor.given("text" => "Will be cast to JSON").as_json.should emit("will", "be", "cast", "to", "json")
+  #   end
+  #
+  #   # Initialize the :tokenizer processor with arguments to test
+  #   # behavior under different conditions.
+  #   it "should output a single record when asked for JSON output" do
+  #     processor(:json => true).given("It was the best of times, it was the worst of times.").should emit(1).records
+  #   end
+  #
+  #   # Initialize processor with arguments and express that the
+  #   # expected output will be in JSON though given as an object.
+  #   it "should output all the tokens for its input record with its JSON output" do
+  #     processor(:json => true).given("You're crazy!").should emit("tokens" => ["youre", "crazy"]).as_json
+  #   end
+  #
+  #   # Initialize processor with arguments, and both input and
+  #   # output will be serialized/deserialized to/from JSON
+  #   # automatically.
+  #   it "can read and write pure JSON" do
+  #     processor(:json => true).given("text" => "You're crazy!").as_json.should emit("tokens" => ["youre", "crazy"]).as_json
+  #   end
+  #
+  #   # Use a processor outside the scope of the top-level :describe
+  #   # block.
+  #   it "has a friend which does the same thing" do
+  #     processor(:similar_tokenizer, :json => true).given("hi there").should emit(2).records
+  #   end
+  module SpecHelpers
+    include ProcessorHelpers
+    include SpecMatchers
+    include IntegrationRunner
+    include IntegrationMatchers
+  end
+
+  Processor.class_eval { include SpecHelpers::ProcessorSpecMethods }
+end

data/lib/wukong/spec_helpers/integration_driver.rb

@@ -0,0 +1,144 @@
+require 'open3'
+
+module Wukong
+  module SpecHelpers
+
+    # Provides a `command` method for writing integration tests for
+    # commands.
+    module IntegrationRunner
+
+      # Spawn a command and capture its STDOUT, STDERR, and exit code.
+      #
+      # The `args` will be joined together into a command line.
+      #
+      # It is expected that you will use the matchers defined in
+      # IntegrationMatchers in your integration tests:
+      #
+      # @example Check output of 'ls' includes a string 'foo.txt'
+      #   it "lists files" do
+      #     command('ls').should have_output('foo.txt')
+      #   end
+      #
+      # @example More complicated
+      #   context "long format" do
+      #     it "lists files with timestamps" do
+      #       command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
+      #     end
+      #   end
+      #
+      # @param [Array<String>] args
+      #
+      # @overload command(*args, options={})
+      #   If the last element of `args` is a Hash it will be used for
+      #   options.
+      #
+      #   The :env option specifies the command line environment to
+      #   use for the command.  By default this will be the value of
+      #   the Ruby process's own `ENV` variable.  If running in a
+      #   context in which the `integration_env` method is defined,
+      #   its return value will be merged on top of `ENV`.  An
+      #   explicitly provided :env option will again be merged on top.
+      #
+      #   The :cwd option specifies the working directory to start in.
+      #   It defaults to the value of <tt>Dir.pwd</tt>
+      #
+      #   @param [Array<String>] args
+      #   @param [Hash] options
+      #   @option options [Hash] env the shell environment to spawn the command with
+      #   @option options [Hash] cwd the directory to execute the command in
+      def command *args
+        a = args.flatten.compact
+        options = (a.last.is_a?(Hash) ? a.pop : {})
+
+        env = ENV.to_hash.dup
+        env.merge!(integration_env) if respond_to?(:integration_env)
+        env.merge!(options[:env] || {})
+
+        cwd   = options[:cwd]
+        cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
+
+        IntegrationDriver.new(a, cwd: cwd, env: env)
+      end
+    end
+
+    # A driver for running commands in a subprocess.
+    class IntegrationDriver
+
+      # The command to execute
+      attr_accessor :cmd
+      
+      # The directory in which to execute the command.
+      attr_accessor :cwd
+
+      # The ID of the spawned subprocess (while it was running).
+      attr_accessor :pid
+
+      # The STDOUT of the spawned process.
+      attr_accessor :stdout
+
+      # The STDERR of the spawned process.
+      attr_accessor :stderr
+
+      # The exit code of the spawned process.
+      attr_accessor :exit_code
+
+      # Run the command and capture its outputs and exit code.
+      #
+      # @return [true, false]
+      def run!
+        return false if ran?
+        Open3.popen3(env, cmd) do |i, o, e, wait_thr|
+          self.pid = wait_thr.pid
+
+          @inputs.each { |input| i.puts(input) }
+          i.close
+
+          self.stdout    = o.read
+          self.stderr    = e.read
+          self.exit_code = wait_thr.value.to_i
+        end
+        @ran = true
+      end
+
+      # Initialize a new IntegrationDriver to run a given command.
+      def initialize args, options
+        @args   = args
+        @env    = options[:env]
+        @cwd    = options[:cwd]
+        @inputs = []
+      end
+
+      def cmd
+        @args.compact.map(&:to_s).join(' ')
+      end
+
+      def on *events
+        @inputs.concat(events)
+        self
+      end
+
+      def env
+        ENV.to_hash.merge(@env || {})
+      end
+
+      def ran?
+        @ran
+      end
+      
+      def cmd_summary
+        [
+         cmd,
+         "with env #{env_summary}",
+         "in dir #{cwd}"
+        ].join("\n")
+      end
+
+      def env_summary
+        { "PATH" => env["PATH"], "RUBYLIB" => env["RUBYLIB"] }.inspect
+      end
+      
+    end
+  end
+end
+
+  

data/lib/wukong/spec_helpers/integration_driver_matchers.rb

@@ -0,0 +1,219 @@
+module Wukong
+  module SpecHelpers
+
+    # Provides matchers for STDOUT, STDERR, and exit code when writing
+    # integration tests for Wukong's command-line APIs.
+    module IntegrationMatchers
+
+      # Checks that each `expectation` appears in the STDOUT of the
+      # command.  Order is irrelevant and each `expectation` can be
+      # either a String to check for inclusion or a Regexp to match
+      # with.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def have_stdout *expectations
+        StdoutMatcher.new(*expectations)
+      end
+
+      # Checks that each `expectation` appears in the STDERR of the
+      # command.  Order is irrelevant and each `expectation` can be
+      # either a String to check for inclusion or a Regexp to match
+      # with.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def have_stderr *expectations
+        StderrMatcher.new(*expectations)
+      end
+
+      # Checks that the command exits with the given `code`.
+      #
+      # @param [Integer] code
+      def exit_with code
+        ExitCodeMatcher.new(code)
+      end
+
+    end
+
+    # A class for running commands and capturing their STDOUT, STDERR,
+    # and exit code.  This class is designed to work with the matchers
+    # defined in IntegrationMatchers.
+    class IntegrationMatcher
+
+      # The driver used to run the actual commands.
+      attr_accessor :driver
+
+      # An array of expectations about the output of the driver.
+      attr_accessor :expectations
+
+      # The expectation which caused failure.
+      attr_accessor :failed_expectation
+
+      # Return whether or not the given command's output matches
+      # expectations.
+      #
+      # If an expectation failes to match, the `failed_expectation`
+      # attribute will be set accordingly.
+      #
+      # @param [IntegrationDriver] driver
+      # @return [true, false]
+      def matches?(driver)
+        self.driver = driver
+        driver.run!
+        expectations.each do |expectation|
+          unless output.send(match_function(expectation), expectation)
+            self.failed_expectation = expectation
+            return false
+          end
+        end
+        true
+      end
+
+      # Create a matcher on the given expectations.  Each expectation
+      # can be either a String or a Regexp.  Strings will be tested
+      # for inclusion in the output, Regexps will be tested for a
+      # match against the output.
+      #
+      # @param [Array<String,Regexp>] expectations
+      def initialize *expectations
+        self.expectations = expectations
+      end
+
+      # :nodoc:
+      def failure_message
+        "Ran\n\n  #{formatted_command}\n\nand expected #{output_description}\n\n#{formatted_output}\n\nto #{match_type}\n\n  #{failed_expectation}"
+      end
+
+      # :nodoc:
+      def negative_failure_message
+        "Expected #{output_description} of #{driver.cmd}\n\n#{output}\n\nto NOT #{match_type}\n\n#{self.failed_expectation}."
+      end
+
+      # :nodoc:
+      def formatted_output
+        output.split("\n").map { |line| '  ' + line }.join("\n")
+      end
+
+      # :nodoc:
+      def formatted_command
+        "$ #{driver.cmd}"
+      end
+
+      # :nodoc:
+      def match_function expectation
+        expectation.is_a?(Regexp) ? :match : :include?
+      end
+
+      # :nodoc:
+      def match_type
+        failed_expectation.is_a?(Regexp) ? 'match' : 'include'
+      end
+
+    end
+
+    # A matcher for the STDOUT of a command.
+    class StdoutMatcher < IntegrationMatcher
+
+      # Picks the STDOUT of the command.
+      def output
+        driver.stdout
+      end
+
+      # :nodoc:
+      def output_description
+        "STDOUT"
+      end
+
+      def description
+        "have the correct #{output_description}"
+      end
+      
+    end
+
+    # A matcher for the STDOUT of a command.
+    class StderrMatcher < IntegrationMatcher
+
+      # Picks the STDOUT of the command.
+      def output
+        driver.stderr
+      end
+
+      # :nodoc:
+      def output_description
+        "STDERR"
+      end
+
+      def description
+        "print an appropriate error message on #{output_description}"
+      end
+    end
+
+    # A matcher for the exit code of a command.
+    class ExitCodeMatcher < IntegrationMatcher
+
+      # Initialize this matcher with the given `code`.
+      #
+      # If `code` is the symbol <tt>:non_zero</tt> then the
+      # expectation will be any non-zero exit code.
+      #
+      # @param [Integer,Symbol] code
+      def initialize code
+        if code ==  :non_zero
+          @expected_code = :non_zero
+        else
+          @expected_code = code.to_i
+        end
+      end
+
+      # Return whether or not the given command's exit code matches
+      # the expectation.
+      #
+      # @param [IntegrationDriver] driver
+      # @return [true, false]
+      def matches?(driver)
+        self.driver = driver
+        driver.run!
+        if non_zero_exit_code?
+          @failed = true if driver.exit_code == 0
+        else
+          @failed = true if driver.exit_code != expected_exit_code
+        end
+        @failed ? false : true
+      end
+
+      # :nodoc:
+      def failure_message
+        "Ran\n\n  #{formatted_command}\n\nexpecting #{expected_exit_code_description}  Got #{driver.exit_code} instead."
+      end
+
+      # :nodoc:
+      def negative_failure_message
+        "Ran\n\n  #{formatted_command}\n\nNOT expecting #{expected_exit_code_description}."
+      end
+
+      # :nodoc:
+      def non_zero_exit_code?
+        @expected_code == :non_zero
+      end
+
+      # :nodoc:
+      def expected_exit_code
+        (@expected_code || 0).to_i
+      end
+
+      # :nodoc:
+      def expected_exit_code_description
+        if non_zero_exit_code?
+          "a non-zero exit code"
+        else
+          "an exit code of #{expected_exit_code}"
+        end
+      end
+
+      # :nodoc:
+      def description
+        "exit with #{expected_exit_code_description}"
+      end
+
+    end
+  end
+end

data/lib/wukong/spec_helpers/processor_helpers.rb

@@ -0,0 +1,95 @@
+module Wukong
+  module SpecHelpers
+    module ProcessorHelpers
+
+      # Creates a new processor in a variety of convenient ways.
+      #
+      # Most simply, called without args, will return a new instance of
+      # a the klass named in the containing `describe` or `context`:
+      #
+      #   context MyApp::Tokenizer do
+      #     it "uses whitespace as the default separator between tokens" do
+      #       processor.separator.should == /\s+/
+      #     end
+      #   end
+      #
+      # if your processor has been registered (you created it with the
+      # <tt>Wukong.processor</tt> helper method or otherwise
+      # registered it yourself) then you can use its name:
+      #
+      #   context :tokenizer do
+      #     it "uses whitespace as the default separator between tokens" do
+      #       processor.separator.should == /\s+/
+      #     end
+      #   end
+      #
+      # The `processor` method can also be used inside RSpec's
+      # `subject` and `let` methods:
+      #
+      #   context "with no arguments" do
+      #     subject { processor }
+      #       it "uses whitespace as the default separator between tokens" do
+      #         separator.should == /\s+/
+      #       end
+      #     end
+      #   end
+      #
+      # and you can easily pass arguments, just like you would on the
+      # command line or in a dataflow definition:
+      # 
+      #   context "with arguments" do
+      #     subject { processor(separator: ' ') }
+      #       it "uses whitespace as the default separator between tokens" do
+      #         separator.should == ' '
+      #       end
+      #     end
+      #   end
+      #
+      # You can even name the processor directly if you want to:
+      #
+      #   context "tokenizers" do
+      #     let(:default_tokenizer) { processor(:tokenizer)                          }
+      #     let(:complex_tokenizer) { processor(:complex_tokenizer, stemming: true)  }
+      #     let(:french_tokenizer)  { processor(:complex_tokenizer, stemming: true)  }
+      #     ...
+      #   end
+      def processor *args, &block
+        case
+        when args.empty?
+          create_processor(self.class.description, {}, &block)
+        when args.first.is_a?(Hash)
+          create_processor(self.class.description, args.first, &block)
+        else
+          create_processor(args[0], (args[1] || {}), &block)
+        end
+      end
+      alias_method :flow, :processor
+
+      # Is the given `klass` a Wukong::Processor?
+      #
+      # @param [Class] klass
+      # @return [true, false]
+      def processor? klass
+        klass.build.is_a?(Processor)
+      end
+
+      # :nodoc:
+      def create_processor name_or_klass, options={}, &block
+        if name_or_klass.is_a?(Class)
+          klass = name_or_klass
+        else
+          klass = Wukong.registry.retrieve(name_or_klass.to_s.to_sym)
+          raise Error.new("Could not find a Wukong::Processor class named '#{name_or_klass}'") if klass.nil?
+        end
+        raise Error.new("#{klass} is not a subclass of Wukong::Processor") unless processor?(klass)
+        settings = Configliere::Param.new
+        Wukong.boot!(settings)
+        proc = klass.build(settings.merge(options))
+        proc.setup
+        proc.instance_eval(&block) if block_given?
+        proc
+      end
+    end
+  end
+end
+