wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/spec_helpers/processor_methods.rb

@@ -0,0 +1,108 @@
+module Wukong
+  module SpecHelpers
+    # This module defines methods to be included into the
+    # Wukong::Processor class.
+    module ProcessorSpecMethods
+
+      # An array of accumulated records to process come match-time.
+      attr_reader :given_records
+
+      # Give a collection of records to the processor.
+      #
+      # @param [Array] records
+      def given *records
+        @given_records ||= []
+        @given_records.concat(records)
+        self                    # for chaining
+      end
+
+      # Give a collection of records to the processor but turn each
+      # to JSON first.
+      #
+      # @param [Array] records
+      def given_json *records
+        self.given(*records.map { |record| MultiJson.dump(record) })
+      end
+      
+      # Give a collection of records to the processor but join each
+      # in a delimited format first.
+      #
+      # @param [Array] records
+      def given_delimited delimiter, *records
+        self.given(*records.map do |record|
+                     record.map(&:to_s).join(delimiter)
+                   end.join("\n"))
+      end
+
+      # Give a collection of records to the processor but join each
+      # in TSV format first.
+      #
+      # @param [Array] records
+      def given_tsv *records
+        self.given_delimited("\t", *records)
+      end
+
+      # Give a collection of records to the processor but join each
+      # in CSV format first.
+      #
+      # @param [Array] records
+      def given_csv *records
+        self.given_delimited(",", *records)
+      end
+
+      # Return the output of the processor on the given records.
+      #
+      # Calling this method, like passing the processor to an `emit`
+      # matcher, will trigger processing of all the given records.
+      #
+      # Returns a SpecDriver, which is a subclass of array, so the
+      # usual matchers like `include` and so on should work, as well
+      # as explicitly indexing to introspect on particular records.
+      #
+      # @return [SpecDriver]
+      def output
+        SpecDriver.new(self).run
+      end
+
+      # Return the output of the processor on the given records,
+      # parsing as a string with the given `delimiter` first.
+      #
+      # @param [String] delimiter
+      # @see #output
+      # @return [Array<String>]
+      def delimited_output(delimiter)
+        output.map { |record| record.split(delimiter) }
+      end
+      
+      # Return the output of the processor on the given records,
+      # parsing as TSV first.
+      #
+      # @see #output
+      # @see #delimited_output
+      # @return [Array<String>]
+      def tsv_output
+        delimited_output("\t")
+      end
+      
+      # Return the output of the processor on the given records,
+      # parsing as CSV first.
+      #
+      # @see #output
+      # @see #delimited_output
+      # @return [Array<String>]
+      def csv_output
+        delimited_output(",")
+      end
+
+      # Return the output of the processor on the given records,
+      # parsing as JSONS first.
+      #
+      # @see #output
+      # @return [Hash,Array]
+      def json_output
+        output.map { |record| MultiJson.load(record) }
+      end
+
+    end
+  end
+end

data/lib/wukong/spec_helpers/shared_examples.rb

@@ -0,0 +1,15 @@
+shared_examples_for 'a processor' do |options={}|
+  name = options[:named]
+  if name
+    it "is registered with the name '#{name}'" do
+      Wukong.registry.retrieve(name.to_sym).should_not be_nil
+    end
+    it{ create_processor(name).should respond_to(:setup)    }
+    it{ create_processor(name).should respond_to(:process)  }
+    it{ create_processor(name).should respond_to(:finalize) }
+    it{ create_processor(name).should respond_to(:stop)     }
+    it{ create_processor(name).should respond_to(:notify)   }
+  else
+    warn "Must supply a name for a processor you want to test"
+  end
+end

data/lib/wukong/spec_helpers/spec_driver.rb

@@ -0,0 +1,28 @@
+module Wukong
+  module SpecHelpers
+    class SpecDriver < Array
+
+      attr_reader :processor
+
+      def initialize processor
+        super()
+        @processor = processor
+      end
+      
+      def run
+        return false unless processor
+        processor.given_records.each do |input|
+          processor.process(input) do |output|
+            self << output
+          end
+        end
+        processor.finalize do |output|
+          self << output
+        end
+        processor.stop
+        self
+      end
+
+    end
+  end
+end

data/lib/wukong/spec_helpers/spec_driver_matchers.rb

@@ -0,0 +1,195 @@
+require_relative('spec_driver')
+
+module Wukong
+  module SpecHelpers
+    
+    module SpecMatchers
+
+      def emit *expected
+        EmitMatcher.new(*expected)
+      end
+
+      def emit_json *expected
+        JsonMatcher.new(*expected)
+      end
+
+      def emit_delimited delimiter, *expected
+        DelimiterMatcher.new(delimiter, *expected)
+      end
+
+      def emit_tsv *expected
+        TsvMatcher.new(*expected)
+      end
+
+      def emit_csv *expected
+        CsvMatcher.new(*expected)
+      end
+    end
+
+    class EmitMatcher
+
+      attr_accessor :driver, :expected, :reason, :expected_record, :actual_record, :mismatched_index
+
+      def matches?(processor)
+        self.driver = SpecDriver.new(processor)
+        driver.run
+        if actual_size != expected_size
+          self.reason = :size
+          return false
+        end
+        return true if just_count?
+        expected.each_with_index do |expectation, index|
+          actual = output[index]
+          if actual != expectation
+            self.reason           = :element
+            self.expected_record  = expectation
+            self.actual_record    = actual
+            self.mismatched_index = index
+            return false
+          end
+        end
+        true
+      end
+      
+      def initialize *expected
+        self.expected = expected
+      end
+
+      def failure_message
+        if reason == :size
+          "Expected #{expected_size} records, got #{actual_size}:\n\n#{pretty_output}"
+        else
+          "Expected the #{ordinalize(mismatched_index)} record to be#{parse_modifier}\n\n#{expected_record}\n\nbut got\n\n#{pretty_output}"
+        end
+      end
+
+      def negative_failure_message
+        if reason == :size
+          "Expected to NOT get #{expected_size} records:\n\n#{output}"
+        else
+          "Expected the #{ordinalize(mismatched_index)} record to NOT be#{parse_modifier}\n\n#{pretty_output}"
+        end
+      end
+      
+      def records
+        @just_count = true
+        self                    # chaining
+      end
+      alias_method :record, :records
+
+      private
+      
+      def just_count?
+        @just_count
+      end
+
+      def actual_size
+        driver.size
+      end
+
+      def expected_size
+        just_count? ? expected.first.to_i : expected.size
+      end
+
+      def output
+        driver
+      end
+
+      def parse_modifier
+      end
+
+      def pretty_output
+        [].tap do |pretty|
+          output.each_with_index do |record, index|
+            s      = (record.is_a?(String) ? record : record.inspect)
+            prefix = case
+            when output.size > 1 && index == mismatched_index
+              " => "
+            when output.size > 1
+              "    "
+            else
+              ''
+            end
+            pretty << [prefix,s].join('')
+          end
+        end.join("\n")
+      end
+
+      # http://stackoverflow.com/questions/1081926/how-do-i-format-a-date-in-ruby-to-include-rd-as-in-3rd
+      def ordinalize array_index
+        n = array_index + 1
+        if (11..13).include?(n % 100)
+          "#{n}th"
+        else
+          case n % 10
+          when 1; "#{n}st"
+          when 2; "#{n}nd"
+          when 3; "#{n}rd"
+          else    "#{n}th"
+          end
+        end
+      end
+    end
+
+    class JsonMatcher < EmitMatcher
+      def output
+        driver.map do |record|
+          begin
+            MultiJson.load(record)
+          rescue => e
+            raise Error.new("Could not parse output of processor as JSON: \n\n#{record}")
+          end
+        end
+      end
+      def parse_modifier
+        ' (after parsing as JSON)'
+      end
+    end
+
+    class DelimitedMatcher < EmitMatcher
+
+      attr_accessor :delimiter
+      
+      def initialize delimiter, *expected
+        self.delimiter = delimiter
+        super(*expected)
+      end
+      
+      def output
+        driver.map do |record|
+          begin
+            record.to_s.split(delimiter)
+          rescue => e
+            raise Error.new("Could not parse as #{delimited_type}': \n\n#{record}")
+          end
+        end
+      end
+
+      def delimited_type
+        "'#{delimiter}-delimited'"
+      end
+      
+      def parse_modifier
+        " (after parsing as #{delimited_type})"
+      end
+    end
+
+    class TsvMatcher < DelimitedMatcher
+      def initialize *expected
+        super("\t", *expected)
+      end
+      def delimited_type
+        "TSV"
+      end
+    end
+
+    class CsvMatcher < DelimitedMatcher
+      def initialize *expected
+        super(",", *expected)
+      end
+      def delimited_type
+        "CSV"
+      end
+    end
+  end
+end

data/lib/wukong/version.rb

@@ -1,3 +1,4 @@
 module Wukong
-  VERSION = '3.0.0.pre'
+  # The current version of Wukong.
+  VERSION = '3.0.0.pre2'
 end

data/lib/wukong/widget/filters.rb

@@ -0,0 +1,311 @@
+module Wukong
+  class Processor
+
+    # A widget which filters input records according to some
+    # criterion.
+    class Filter < Processor
+
+      # Process a `record` by yielding it only if it should be
+      # selected by this filter.
+      #
+      # @param [Object] record an input record
+      # @yield [record] yielded if this record should pass the filter
+      # @yieldparam [Object] record
+      # @see #select?
+      # @see #reject?
+      def process(record)
+        yield(record) if select?(record)
+      end
+
+      # Should the given `record` be passed by this filter?
+      #
+      # @param [Object] record
+      # @return [true, false]
+      # @see #reject?
+      def select?(record)
+        true
+      end      
+
+      # Should the given `record` be rejected by this filter?
+      #
+      # @param [Object] record
+      # @return [true, false]
+      # @see #select?
+      def reject?(record)
+        not select?(record)
+      end
+
+      register
+    end
+
+    # A widget which passes all records, i.e. - it acts just like
+    # `cat`.
+    #
+    # @example Pass all records unmodified on the command line
+    #
+    #   $ cat input
+    #   1
+    #   2
+    #   3
+    #   $ cat input | wu-local identity
+    #   1
+    #   2
+    #   3
+    #   
+    # @example Pass all records unmodified in a dataflow
+    #
+    #   Wukong.dataflow(:uses_identity) do
+    #     ... | identity | ...
+    #   end
+    #   
+    # @see Filter
+    # @see Null
+    class Identity < Filter
+      register
+    end
+    
+    # A widget which doesn't pass any records, i.e. - it acts just
+    # like <tt>/dev/null</tt>.
+    #
+    # @example Filter all records on the command line
+    #
+    #   $ cat input
+    #   1
+    #   2
+    #   3
+    #   $ cat input | wu-local null
+    #   
+    # @example Filter all records from a dataflow
+    #
+    #   Wukong.dataflow(:uses_null) do
+    #     ... | null | ...
+    #   end
+    #   
+    # @see Filter
+    # @see All
+    class Null < Filter
+      # Prevents any records from passing because it always returns
+      # `false`.
+      #
+      # @param [Object] record
+      # @return false
+      def select? record
+        false
+      end
+      register
+    end
+
+    # A widget which only passes records if they match a regular
+    # expression.
+    #
+    # @example Passing records which match a given expression on the command-line
+    #
+    #   $ cat input
+    #   apple
+    #   banana
+    #   cat
+    #   $ cat input | wu-local regexp --match='^a'
+    #   apple
+    #
+    # @example Passing records which match a given expression in a dataflow
+    #
+    #   Wukong.dataflow(:uses_regexp) do
+    #     ... | regexp(match: /^a/) | ...
+    #   end
+    #
+    # @see Filter
+    # @see NotRegexpFilter
+    class RegexpFilter < Filter
+
+      # The regular expression to use to match records.
+      field :match, Regexp
+      
+      # Selects a `record` only if it matches this widget's `match`
+      # field.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        return true unless match
+        match =~ record.to_s
+      end
+      register(:regexp)
+    end
+
+    # A widget which only passes records if they *don't* match a
+    # regular expression.
+    #
+    # @example Passing records which don't match a given expression on the command-line
+    #
+    #   $ cat input
+    #   apple
+    #   banana
+    #   cat
+    #   $ cat input | wu-local not_regexp --match='^a'
+    #   banana
+    #   cat
+    #
+    # @example Passing records which don't match a given expression in a dataflow
+    #
+    #   Wukong.dataflow(:uses_not_regexp) do
+    #     ... | not_regexp(match: /^a/) | ...
+    #   end
+    #
+    # @see Filter
+    # @see NotRegexpFilter
+    class NotRegexpFilter < RegexpFilter
+      # Select a `record` only if it <b>doesn't</b> match this
+      # widget's `match` field.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        return true unless match
+        not match =~ record.to_s
+      end
+      register(:not_regexp)      
+    end
+
+    # A widget which only lets a certain number of records through.
+    #
+    # @example Letting the first 3 records through on the command-line
+    #
+    #   $ cat input
+    #   1
+    #   2
+    #   3
+    #   4
+    #   $ cat input | wu-local limit --max=3
+    #   1
+    #   2
+    #   3
+    #
+    # @example Letting the first 3 records through in a dataflow
+    #
+    #   Wukong.dataflow(:uses_limit) do
+    #     ... | limit(max: 3) | ...
+    #   end
+    #
+    # @see Filter
+    class Limit < Filter
+
+      # The maximum number of records to let pass.
+      field :max, Integer, :default => Float::INFINITY
+
+      # The current record count.
+      attr_accessor :count
+
+      # Initializes the record count to zero.
+      def setup
+        self.count = 0
+      end
+
+      # Select a record only if we're below the max count.  Increments
+      # the count for this widget.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        keep = @count < max
+        @count += 1
+        keep
+      end
+      register
+    end
+
+    # A widget which samples a certain fraction of input records.
+    #
+    # @example Sampling records on the command line
+    #
+    #   $ cat input
+    #   1
+    #   2
+    #   3
+    #   4
+    #   $ cat input | wu-local sample --fraction=0.5
+    #   1
+    #   3
+    #
+    # @example Sampling records in a dataflow
+    #
+    #   Wukong.dataflow(:uses_sample) do
+    #     ... | sample(fraction: 0.5) ...
+    #   end
+    #
+    # @see Filter
+    # @see Limit
+    class Sample < Filter
+
+      # The fraction of records to let pass.  Must be between 0.0 and
+      # 10.0
+      field :fraction, Float, :default => 1.0
+
+      # Selects a `record` randomly, with a probability given the the
+      # `fraction` for this widget.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        rand() < fraction
+      end
+      register
+    end
+
+    # A widget useful for creating filters on the fly in a dataflow.
+    #
+    # When writing a filtering processor out as a class, just use the
+    # DSL for creating processors:
+    #
+    # @example Creating a select filter the usual way
+    #
+    #   Wukong.processor(:my_filter, Wukong::Processor::Filter) do
+    #     def select? record
+    #       record.length > 3
+    #     end
+    #   end
+    #
+    # When in a dataflow, sometimes it's easier to create a processor
+    # like this on the fly.
+    #
+    # @example Creating a select filter on the fly in a dataflow
+    #
+    #   Wukong.dataflow(:my_flow) do
+    #     ... | select { |record| record.length > 3 } | ...
+    #   end
+    #
+    # @see Filter
+    # @see Reject
+    class Select < Filter
+
+      # Selects the given `record` by delegating to the
+      # `perform_action` method, which will automatically be
+      # populating by the block used to create this filter in the
+      # dataflow DSL.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      # @see Processor#perform_action
+      def select?(record)
+        perform_action(record)
+      end
+      register
+    end
+
+    # A widget useful for creating filters on the fly in a dataflow.
+    #
+    # @see Select
+    class Reject < Filter
+      # Rejects the given `record` by delegating to the
+      # `perform_action` method.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      # @see Processor#perform_action
+      def select?(record)
+        not perform_action(record)
+      end
+      register
+    end
+
+  end
+end