wukong 3.0.1 → 4.0.0

data/lib/wukong.rb

@@ -1,5 +1,4 @@
 require 'configliere'
-require 'vayacondios-client'
 require 'multi_json'
 require 'eventmachine'
 require 'log4r'
@@ -35,6 +34,10 @@ module Wukong
   
   add_shortcut_method_for(:processor, ProcessorBuilder)
   add_shortcut_method_for(:dataflow,  DataflowBuilder)
+
+  def self.doc_helpers_path
+    File.expand_path('../wukong/doc_helpers.rb', __FILE__)
+  end
   
 end
 
@@ -45,5 +48,7 @@ require_relative 'wukong/widgets'
 require_relative 'wukong/local'
 
 module Wukong
+
+  # Built-in Wukong processors and dataflows.
   BUILTINS = Set.new(Wukong.registry.show.keys)
 end

data/lib/wukong/dataflow.rb

@@ -1,5 +1,12 @@
 module Wukong
-  class DataflowBuilder < Hanuman::GraphBuilder
+
+  class Dataflow < Hanuman::Tree
+    def self.configure(settings)
+      settings.description = builder.description if builder.description
+    end
+  end
+  
+  class DataflowBuilder < Hanuman::TreeBuilder
 
     def description desc=nil
       @description = desc if desc
@@ -10,17 +17,20 @@ module Wukong
 
     def handle_dsl_arguments_for(stage, *args, &action)
       options = args.extract_options!
+      while stages.include?(stage.label)
+        parts = stage.label.to_s.split('_')
+        if parts.last.to_i > 0
+          parts[-1] = parts.last.to_i + 1
+        else
+          parts.push(1)
+        end
+        stage.label = parts.map(&:to_s).join('_').to_sym
+      end
       stage.merge!(options.merge(action: action).compact)
-      stage      
+      stage.graph = self
+      stage
     end
   
-    def linkable_name(direction) 
-      case direction
-      when :in  then directed_sort.first
-      when :out then directed_sort.last
-      end
-    end
-
     def method_missing(name, *args, &blk)
       if stages[name]
         handle_dsl_arguments_for(stages[name], *args, &blk)
@@ -30,43 +40,4 @@ module Wukong
     end
     
   end
-  
-  class Dataflow < Hanuman::Graph
-
-    def self.description desc=nil
-      @description = desc if desc
-      @description
-    end
-    
-    def has_input?(stage)
-      links.any?{ |link| link.into == stage }
-    end
-    
-    def has_output?(stage)
-      links.any?{ |link| link.from == stage }
-    end
-
-    def connected?(stage)
-      input  = has_input?(stage)  || stages[stage].is_a?(Wukong::Source)
-      output = has_output?(stage) || stages[stage].is_a?(Wukong::Sink)
-      input && output
-    end
-
-    def complete?
-      stages.all?{ |(name, stage)| connected? name }
-    end
-
-    def setup
-      directed_sort.each{ |name| stages[name].setup }
-    end
-
-    def run
-      stages[directed_sort.first].run
-    end
-
-    def stop
-      directed_sort.each{ |name| stages[name].stop }
-    end
-
-  end
 end

data/lib/wukong/driver.rb

@@ -1,103 +1,214 @@
+require_relative('driver/wiring')
+
 module Wukong
+
+  # A Driver is a class including the DriverMethods module which
+  # connects a Dataflow or Processor to the external world of inputs
+  # and outputs.
+  #
+  # @example Minimal Driver class
+  #
+  #   class MinimalDriver
+  #     include Wukong::DriverMethods
+  #     def initialize(label, settings)
+  #       construct_dataflow(label, settings)
+  #     end
+  #     def process record
+  #       puts record
+  #     end
+  #   end
+  #
+  # The MinimalDriver#send_through_dataflow method can be called on an
+  # instance of MinimalDriver with any input record.
+  #
+  # This record will be passed through the dataflow, starting from its
+  # root, and each record yielded at the leaves of the dataflow will
+  # be passed to the driver's #process method.
+  #
+  # The #process method of an implementing driver should *not* yield,
+  # unlike the process method of a Processor class.  Instead, it
+  # should treat its argument as an output of the dataflow and do
+  # something appropriate to the driver (write to file, database,
+  # terminal, &c.).
+  #
+  # Drivers are also responsible for implementing the lifecycle of
+  # processors and dataflows they drive.  A more complete version of
+  # the above driver class would:
+  #
+  #   * call the #setup_dataflow method when ready to trigger the
+  #     Processor#setup method on each processor in the dataflow
+  #
+  #   * call the #finalize_dataflow method when indicating that the
+  #     dataflow should consider a batch of records complete
+  #
+  #   * call the #finalize_and_stop_dataflow method to indicate the
+  #     last batch of records and to trigger the Processor#stop method
+  #     on each processor in the dataflow
+  #
+  # Driver instances are started by Runners which should delegate to
+  # the `start` method driver class itself.
+  # 
+  # @see Wukong::Local::StdioDriver for a complete example of a driver.
+  # @see Wukong::Local::Runner for an example of how runners call drivers.
   module DriverMethods
 
-    attr_accessor :dataflow
-    
+    attr_accessor :label
     attr_accessor :settings
+    attr_accessor :dataflow
 
-    def driver
-      @driver ||= Driver.new(dataflow)
+    # Classes including DriverMethods should override this method with
+    # some way of handling the `output_record` that is appropriate for
+    # the driver.
+    #
+    # @param [Object] output_record
+    def process output_record
+      raise NotImplementedError.new("Define the #{self.class}#process method to handle output records from the dataflow")
     end
 
-    def lookup(label)
-      raise Wukong::Error.new("could not find definition for <#{label}>") unless Wukong.registry.registered?(label.to_sym)
-      Wukong.registry.retrieve(label.to_sym)
+    # Construct a dataflow from the given `label` and `settings`.
+    #
+    # This method does **not** cause Processor#setup to be called on
+    # any of the processors in the dataflow.  Call the #setup_dataflow
+    # method to explicitly have setup occur.  This distinction is
+    # useful for drivers which themselves need to do complex
+    # initialization before letting processors in the dataflow
+    # initialize.
+    #
+    # @param [Symbol] label the name of the dataflow (or processor) to build
+    # @param [Hash] settings
+    # @param settings [String] :to Serialize all output via the named serializer (json, tsv)
+    # @param settings [String] :from Deserialize all input via the named deserializer (json, tsv)
+    # @param settings [String] :as Recordize each input as instances of the given class
+    # 
+    # @see #setup_dataflow
+    def construct_dataflow(label, settings={})
+      self.label    = label
+      self.settings = settings
+      prepend(:recordize)                       if settings[:as]
+      prepend("from_#{settings[:from]}".to_sym) if settings[:from]
+      append("to_#{settings[:to]}".to_sym)      if settings[:to]
+      build_dataflow
     end
-    
-    def lookup_and_build(label, options = {})
-      lookup(label).build(options)
-    end
-    
-    def build_serializer(direction, label, options)
-      lookup_and_build("#{direction}_#{label}", options)
+
+    # Set up this driver.  Called before setting up any of the
+    # dataflow stages.
+    def setup
     end
 
-    def add_serialization(dataflow, direction, label, options)
-      case direction
-      when :to   then dataflow.push    build_serializer(direction, label, options)
-      when :from then dataflow.unshift build_serializer(direction, label, options)
+    # Walks the dataflow and calls Processor#setup on each of the
+    # processors.
+    def setup_dataflow
+      setup
+      dataflow.each_stage do |stage|
+        stage.setup
       end
     end
 
-    def setup_dataflow
-      dataflow.each(&:setup)
+    # Send the given `record` through the dataflow.
+    #
+    # @param [Object] record
+    def send_through_dataflow(record)
+      wiring.start_with(dataflow.root).call(record)
     end
 
+    # Perform finalization code for this driver.  Runs after #setup
+    # and before #stop.
+    def finalize
+    end
+
+    # Indicate a full batch of records has already been sent through
+    # and any batch-oriented or accumulative operations should trigger
+    # (e.g. - counting).
+    #
+    # Walks the dataflow calling Processor#finalize on each processor.
+    #
+    # On the *last* batch, the #finalize_and_stop_dataflow method
+    # should be called instead.
+    #
+    # @see #finalize_and_stop_dataflow
     def finalize_dataflow
-      dataflow.each do |stage|
-        stage.finalize(&driver.advance(stage)) if stage.respond_to?(:finalize)
+      finalize
+      dataflow.each_stage do |stage|
+        stage.finalize(&wiring.advance(stage))
       end
     end
 
+    # Works similar to #finalize_dataflow but calls Processor#stop
+    # after calling Processor#finalize on each processor.
     def finalize_and_stop_dataflow
-      dataflow.each do |stage|
-        stage.finalize(&driver.advance(stage)) if stage.respond_to?(:finalize)
+      finalize
+      dataflow.each_stage do |stage|
+        stage.finalize(&wiring.advance(stage))
         stage.stop
-      end      
+      end
+      stop
     end
 
-    # So pretty...
-    def construct_dataflow(label, options)
-      dataflow = lookup_and_build(label, options)
-      dataflow = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
-      expected_input_model  = (options[:consumes].constantize rescue nil)    || dataflow.first.expected_record_type(:consumes)
-      dataflow.unshift lookup_and_build(:recordize, model: expected_input_model)  if expected_input_model
-      expected_output_model = (options[:produces].constantize rescue nil)    || dataflow.first.expected_record_type(:produces)
-      dataflow.push lookup_and_build(:recordize, model: expected_output_model)    if expected_output_model
-      expected_input_serialization  = options[:from] || dataflow.last.expected_serialization(:from)
-      add_serialization(dataflow, :from, expected_input_serialization, options)   if expected_input_serialization
-      expected_output_serialization = options[:to]   || dataflow.last.expected_serialization(:to)
-      add_serialization(dataflow, :to, expected_output_serialization, options)    if expected_output_serialization
-      dataflow.push self
-    end    
-  end
+    # Perform shutdown code for this driver.  Called after #finalize
+    # and after all stages have been finalized and stopped.
+    def stop
+    end
 
-  class Driver
-    attr_accessor :dataflow
+    protected
 
-    def initialize(dataflow)
-      @dataflow = dataflow
+    # The builder for this driver's `label`, either for a Processor or
+    # a Dataflow.
+    #
+    # @return [Wukong::ProcessorBuilder, Wukong::DataflowBuilder]
+    def builder
+      return @builder if @builder
+      raise Wukong::Error.new("could not find definition for <#{label}>") unless Wukong.registry.registered?(label.to_sym)
+      @builder = Wukong.registry.retrieve(label.to_sym)
     end
 
-    def to_proc
-      return @wiring if @wiring
-      @wiring = Proc.new do |stage, record|
-        stage.process(record, &advance(stage)) if stage          
-      end
+    # Return the builder for this driver's dataflow.
+    #
+    # Even if a Processor was originally named by this driver's
+    # `label`, a DataflowBuilder will be returned here.  The
+    # DataflowBuilder is itself built from just the ProcessorBuilder
+    # alone.
+    #
+    # @return [Wukong::DataflowBuilder]
+    # @see #builder
+    def dataflow_builder
+      @dataflow_builder ||= (builder.is_a?(DataflowBuilder) ? builder : Wukong::DataflowBuilder.receive(for_class: Class.new(Wukong::Dataflow), stages: {label.to_sym => builder}))
     end
 
-    def send_through_dataflow(record)
-      start_with(dataflow.first).call(record)
-    end
-    
-    def start_with(stage)
-      to_proc.curry.call(stage)
+    # Build the dataflow using the #dataflow_builder and the supplied
+    # `settings`.
+    #
+    # @return [Wukong::Dataflow]
+    def build_dataflow
+      self.dataflow = dataflow_builder.build(settings)
     end
 
-    def advance(stage)
-      next_stage = stage_iterator(stage)
-      start_with(next_stage)
+    # Add the processor with the given `new_label` in front of this
+    # driver's dataflow, making it into the new root of the dataflow.
+    #
+    # @param [Symbol] new_label
+    def prepend new_label
+      raise Wukong::Error.new("could not find processor <#{new_label}> to prepend") unless Wukong.registry.registered?(new_label)
+      dataflow_builder.prepend(Wukong.registry.retrieve(new_label))
     end
 
-    # This should properly be defined on dataflow/builder
-    def stage_iterator(stage)
-      position = dataflow.find_index(stage)
-      dataflow[position + 1]    
-    end 
+    # Add the processor with the given `new_label` at the end of each
+    # of this driver's dataflow's leaves.
+    #
+    # @param [Symbol] new_label
+    def append new_label
+      raise Wukong::Error.new("could not find processor <#{new_label}> to append") unless Wukong.registry.registered?(new_label)
+      dataflow_builder.append(Wukong.registry.retrieve(new_label))
+    end
 
-    def call(*args)
-      to_proc.call(*args)
+    # Returns the underlying Wiring object that will coordinate
+    # transfer of records from the driver to the dataflow and back to
+    # the driver.
+    #
+    # @return [Wiring]
+    def wiring
+      @wiring ||= Wiring.new(self, dataflow)
     end
     
   end
+
 end

data/lib/wukong/{local → driver}/event_machine_driver.rb

@@ -1,12 +1,7 @@
 module Wukong
-
-  # A module which can be included by other drivers which lets them
-  # use EventMachine under the hood.
   module EventMachineDriver
-    
     include DriverMethods
 
-    # :nodoc:
     def self.included klass
       klass.class_eval do
         def self.add_signal_traps
@@ -15,13 +10,6 @@ module Wukong
         end
       end
     end
-
-    # :nodoc:
-    def initialize(label, settings)
-      super
-      @settings = settings      
-      @dataflow = construct_dataflow(label, settings)
-    end
-    
+      
   end
 end

data/lib/wukong/driver/wiring.rb

@@ -0,0 +1,68 @@
+module Wukong
+
+  # Provides a very Ruby-minded way of walking a dataflow connected to
+  # a driver.
+  class Wiring
+
+    # The driver instance that likely calls the #start_with method and
+    # provides a #process method to be called by this wiring.
+    attr_accessor :driver
+
+    # The dataflow being wired.
+    attr_accessor :dataflow
+
+    # Construct a new Wiring for the given `driver` and `dataflow`.
+    #
+    # @param [#process] driver
+    # @param [Wukong::Dataflow] dataflow
+    def initialize(driver, dataflow)
+      @driver    = driver
+      @dataflow  = dataflow
+    end
+
+    # Return a proc which, if called with a record, will process that
+    # record through each of the given `stages` as well as through the
+    # rest of the dataflow ahead of them.
+    #
+    # @param [Array<Wukong::Stage>] stages
+    # @return [Proc]
+    def start_with(*stages)
+      to_proc.curry.call(stages)
+    end
+
+    # Return a proc (the output of #start_with) which will process
+    # records through the stages that are ahead of the given stage.
+    #
+    # @param [Wukong::Stage] stage
+    # @return [Proc]
+    #
+    # @see #start_with
+    def advance(stage)
+      # This is where the tree of procs will terminate, but only after
+      # having passed all output records through the driver -- the
+      # last "stage".
+      return start_with() if stage.nil? || stage == driver
+
+      # Otherwise we're still in the middle of the tree...
+      descendents = dataflow.descendents(stage)
+      if descendents.empty?
+        # No descendents it means we've reached a leaf of the tree so
+        # we'll run records through the driver to generate output.
+        start_with(driver)
+      else
+        # Otherwise continue down the tree of procs...
+        start_with(*descendents)
+      end
+    end
+    
+    # :nodoc:
+    def to_proc
+      return @wiring if @wiring
+      @wiring = Proc.new do |stages, record|
+        stages.each do |stage|
+          stage.process(record, &advance(stage)) if stage
+        end
+      end
+    end
+  end
+end