wukong 3.0.1 → 4.0.0

data/lib/wukong/spec_helpers/shared_examples.rb

@@ -6,7 +6,6 @@ shared_examples_for 'a processor' do |options = {}|
   it{ processor(options[:named]).processor.should respond_to(:process)  }
   it{ processor(options[:named]).processor.should respond_to(:finalize) }
   it{ processor(options[:named]).processor.should respond_to(:stop)     }
-  it{ processor(options[:named]).processor.should respond_to(:notify)   }
 end
 
 shared_examples_for 'a plugin' do |options = {}|

data/lib/wukong/spec_helpers/unit_tests.rb

@@ -99,13 +99,14 @@ module Wukong
       #     let(:french_tokenizer)  { processor(:complex_tokenizer, stemming: true, language: 'fr')  }
       #     ...
       #   end
-      def unit_test_runner *args
+      def unit_test_runner *args, &block
         settings = args.extract_options!
         name     = (args.first || self.class.description)
-        runner   = UnitTestRunner.new(name, settings)
-        yield runner.driver.processor if block_given?
-        runner.boot!
-        runner.driver
+        UnitTestRunner.new(name, settings).tap do |the_runner|
+          the_runner.program_name = 'wu-local'
+          yield the_runner.driver.processor if block_given?
+          the_runner.boot!(settings)
+        end.driver
       end
       alias_method :processor, :unit_test_runner
 

data/lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb

@@ -4,22 +4,12 @@ module Wukong
       
       include Wukong::DriverMethods
       
-      def initialize label, settings
+      def initialize label, settings={}
         super()
-        @settings = settings
-        @dataflow = construct_dataflow(label, settings)
+        construct_dataflow(label, settings)
         setup_dataflow
       end
       
-      def setup
-      end
-      
-      def finalize
-      end
-      
-      def stop
-      end
-      
       def process output
         self << output
       end
@@ -27,14 +17,14 @@ module Wukong
       def run
         return false unless dataflow
         given_records.each do |input|
-          driver.send_through_dataflow(input)
+          send_through_dataflow(input)
         end
         finalize_and_stop_dataflow
         self
       end
       
       def processor
-        dataflow.first
+        dataflow.root
       end
       
       # An array of accumulated records to process come match-time.

data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb

@@ -19,29 +19,28 @@ module Wukong
     #   of the unit test back into the test suite
     class UnitTestRunner < Wukong::Local::LocalRunner
 
-      # The processor this runner will create in the same way as
-      # `wu-local`.
-      attr_accessor :processor
-
       # Initialize a new UnitTestRunner for the processor with the
       # given `label` and `settings`.
       #
       # @param [Symbol] label
       # @param [Hash] settings
-      def initialize label, settings
-        self.processor = label
+      def initialize label, settings={}
+        @dataflow = label
         params = Configliere::Param.new
-        params.use(:commandline)
         params.merge!(settings)
         super(params)
       end
 
+      def dataflow
+        @dataflow
+      end
+
       # Override the LocalDriver with the UnitTestDriver so we can
       # more easily pass in and retrieve processed records.
       #
       # @return [UnitTestDriver]
       def driver
-        @driver ||= UnitTestDriver.new(processor, settings)
+        @driver ||= UnitTestDriver.new(dataflow, settings)
       end
 
       # No need to load commandline arguments when we are testing

data/lib/wukong/version.rb

@@ -1,3 +1,3 @@
 module Wukong
-  VERSION = '3.0.1'
+  VERSION = '4.0.0'
 end

data/lib/wukong/widget/echo.rb

@@ -0,0 +1,55 @@
+module Wukong
+  class Processor
+
+    # A widget that yields whatever you instantiate it with.
+    #
+    # This is most useful when you have a small but predictable input
+    # that you don't want or can't pass via usual input channels like
+    # STDIN.
+    #
+    # @example Works just like you think on the command line
+    #
+    #   $ echo something else | wu-local echo --input=hello
+    #   hello
+    #
+    # @example Pass some fixed input to your downstream code.
+    #
+    #   # my_flow.rb
+    #   Wukong.dataflow(:my_flow) do
+    #     echo(input: {key: 'value'}) | my_proc | ...
+    #   end
+    #
+    # This differs from from the `:identity` processor because it
+    # doesn't pass on what it receives but what you instantiate it
+    # with.
+    #
+    # @see Identity
+    class Echo < Processor
+
+      description <<EOF
+A widget that yields whatever you instantiate it with.
+
+This is most useful when you have a small but predictable input
+that you don't want or can't pass via usual input channels like
+STDIN.
+
+Works just like you think on the command line (the process won't terminate)
+
+  $ echo something else | wu-local echo --input=hello
+  hello
+EOF
+
+      field :input, Whatever, :default => nil, :doc => "The record to echo"
+
+      # Yields the `input` no matter what you pass it.
+      #
+      # @param [Object] _ the new input record which is ignored
+      # @yield [input]
+      # @yieldparam [Object] input the original input
+      def process _
+        yield input
+      end
+      register
+    end
+  end
+end

data/lib/wukong/widget/{processors.rb → extract.rb}

@@ -3,58 +3,6 @@ require_relative('utils')
 module Wukong
   class Processor
 
-    # A widget that will log all incoming records.
-    #
-    # @example Logging records from the command line
-    #
-    #   $ cat input
-    #   1
-    #   2
-    #   3
-    #   $ cat input | wu-local logger
-    #   2012-11-28 18:20:46 [INFO] Logger: 1
-    #   2012-11-28 18:20:46 [INFO] Logger: 2
-    #   2012-11-28 18:20:46 [INFO] Logger: 3
-    #
-    # @example Logging records within a dataflow
-    #
-    #   Wukong.dataflow(:uses_logger) do
-    #     ... | logger
-    #   end
-    class Logger < Processor
-      field :level, Symbol, :default => :info, :doc => "Log level priority"
-
-      description <<EOF
-This processor passes all input records unmodified, making a log
-statement on each one.
-
-  $ cat input
-  1
-  2
-  3
-  $ cat input | wu-local logger
-  INFO 2013-01-04 17:10:59 [Logger              ] -- 1
-  INFO 2013-01-04 17:10:59 [Logger              ] -- 2
-  INFO 2013-01-04 17:10:59 [Logger              ] -- 3
-
-You can set the priority level of the log messages with the --level
-flag.
-
-  $ cat input | wu-local logger --level=debug
-  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 1
-  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 2
-  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 3
-EOF
-
-      # Process a given `record` by logging it.
-      #
-      # @param [Object] record
-      def process(record)
-        log.send(level, record)
-      end
-      register
-    end
-
     # A widget that extracts parts of incoming records.
     #
     # This widget can extract part of the following kinds of objects:
@@ -170,59 +118,5 @@ EOF
       end
       register
     end
-    
-    class Topic < Processor
-      
-      field :topic, Symbol, :doc => "Topic to label the record with"
-      
-      def process(record)
-        yield perform_action(record)
-      end
-
-      def perform_action(record)
-        assign_topic(record, topic)
-      end      
-
-      def assign_topic(record, topic_name)
-        record.define_singleton_method(:topic){ topic_name }
-        record
-      end
-      register
-    end
-    
-    # Until further notice, this processor is unusable due to the invocation of yield
-    # class Foreach < Processor
-    #   def process(record, &blk)
-    #     perform_action(record, &blk)
-    #   end
-    #   register
-    # end
-
-    class Map < Processor
-      def process(record)
-        yield perform_action(record)
-      end
-      register
-    end
-    
-    class Flatten < Processor
-      def process(records)
-        records.respond_to?(:each) ? records.each{ |record| yield(record) } : yield(records)
-      end
-      register
-    end
-
-    # Mixin processor behavior
-    module BufferedProcessor
-      def setup()                           ; end
-      def process(record) @buffer << record ; end
-      def stop()                            ; end
-    end
-  
-    module StdoutProcessor
-      def setup()         $stdout.sync        ; end
-      def process(record) $stdout.puts record ; end
-      def stop()                              ; end
-    end
   end
 end

data/lib/wukong/widget/filters.rb

@@ -432,6 +432,21 @@ EOF
       end
       register
     end
+
+    # Select a record only if it is non-nil.
+    #
+    # @see Filter
+    class Compact < Filter
+
+      # Select a record only if it is non-nil.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        ! record.nil?
+      end
+      register
+    end
     
   end
 end

data/lib/wukong/widget/logger.rb

@@ -0,0 +1,56 @@
+module Wukong
+  class Processor
+
+    # A widget that will log all incoming records.
+    #
+    # @example Logging records from the command line
+    #
+    #   $ cat input
+    #   1
+    #   2
+    #   3
+    #   $ cat input | wu-local logger
+    #   2012-11-28 18:20:46 [INFO] Logger: 1
+    #   2012-11-28 18:20:46 [INFO] Logger: 2
+    #   2012-11-28 18:20:46 [INFO] Logger: 3
+    #
+    # @example Logging records within a dataflow
+    #
+    #   Wukong.dataflow(:uses_logger) do
+    #     ... | logger
+    #   end
+    class Logger < Processor
+      field :level, Symbol, :default => :info, :doc => "Log level priority"
+
+      description <<EOF
+This processor passes all input records unmodified, making a log
+statement on each one.
+
+  $ cat input
+  1
+  2
+  3
+  $ cat input | wu-local logger
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 1
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 2
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 3
+
+You can set the priority level of the log messages with the --level
+flag.
+
+  $ cat input | wu-local logger --level=debug
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 1
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 2
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 3
+EOF
+
+      # Process a given `record` by logging it.
+      #
+      # @param [Object] record
+      def process(record)
+        log.send(level, record)
+      end
+      register
+    end
+  end
+end

data/lib/wukong/widget/operators.rb

@@ -0,0 +1,82 @@
+module Wukong
+  class Processor
+
+    # Yield the result of this processor's action for each input
+    # record.
+    #
+    # @example Apply a function (like a parser) to each record
+    #
+    #   Wukong.dataflow(:parser) do
+    #     ... |  map { |string| MyParser.parse(string) } | ...
+    #   end
+    #
+    # @example Succintly map between objects
+    #
+    #   Wukong.dataflow(:converter) do
+    #     ... | my_book_parser | map(&:author) | author_processor | ...
+    #   end
+    #
+    # Can also be called with the :compact option which will check if
+    # the result of the action is non falsy before yielding.
+    #
+    # @example Mapping but only if it exists
+    #
+    #   Wukong.dataflow(:converter_and_trimmer) do
+    #     ... | my_book_parser | map(compact: true, &:author) | processor_that_needs_an_author | ...
+    #   end
+    class Map < Processor
+
+      field :compact, :boolean, default: false
+
+      # Call #perform_action on the input_record and yield the
+      # returned output record.
+      #
+      # If #compact then only yield the output record if it is not
+      # falsy.
+      #
+      # @param [Object] input_record
+      # @yield [output_record] if compact, then only yield if it is not falsy
+      # @yieldparam [Object] output_record the result of #perform_action
+      #
+      # @see Flatten
+      def process(input_record)
+        output_record = perform_action(input_record)
+        if compact
+          yield output_record if output_record
+        else
+          yield output_record
+        end
+      end
+      register
+    end
+
+    # If an input record defines the #each method then yield each of
+    # its records.  Otherwise yield the input record.
+    #
+    # @example Turning one record into many
+    #
+    #   Wukong.dataflow(:authors_to_books) do
+    #     ... | author_parser | map(&:books) | flatten | book_processor | ...
+    #   end
+    #
+    # @see Map
+    class Flatten < Processor
+
+      # If input_record responds to #each then yield each of these as
+      # an output record.  Else, just yield the input_record.
+      #
+      # @param [Object] input_record
+      # @yield [output_record]
+      # @yieldparam [Object] output_record
+      def process(input_record)
+        if input_record.respond_to?(:each)
+          input_record.each{ |output_record| yield(output_record) }
+        else
+          yield(input_record)
+        end
+      end
+      register
+    end
+
+  end
+end