ul-wukong 4.1.0

data/lib/wukong/doc_helpers/processor_handler.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module DocHelpers
+
+    # Handles the Wukong.processor syntax.
+    class ProcessorHandler < YARD::Handlers::Ruby::ClassHandler
+
+      handles method_call(:processor)
+
+      # :nodoc:
+      def base_processor_class
+        @base_processor_class ||= YARD::CodeObjects::ClassObject.new(namespace, "Wukong::Processor")
+      end
+
+      # :nodoc:
+      def process
+        processor_name  = statement.parameters.first.jump(:tstring_content, :ident).source
+        class_name      = Gorillib::Inflector.camelize(processor_name)
+        processor_class = create_class(class_name, base_processor_class)
+        processor_body  = statement.last.last
+        
+        push_state(:owner => processor_class, :scope => :class, :namespace => processor_class) do
+          parse_block(processor_body)
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/wukong/driver.rb

@@ -0,0 +1,214 @@
+require_relative('driver/wiring')
+
+module Wukong
+
+  # A Driver is a class including the DriverMethods module which
+  # connects a Dataflow or Processor to the external world of inputs
+  # and outputs.
+  #
+  # @example Minimal Driver class
+  #
+  #   class MinimalDriver
+  #     include Wukong::DriverMethods
+  #     def initialize(label, settings)
+  #       construct_dataflow(label, settings)
+  #     end
+  #     def process record
+  #       puts record
+  #     end
+  #   end
+  #
+  # The MinimalDriver#send_through_dataflow method can be called on an
+  # instance of MinimalDriver with any input record.
+  #
+  # This record will be passed through the dataflow, starting from its
+  # root, and each record yielded at the leaves of the dataflow will
+  # be passed to the driver's #process method.
+  #
+  # The #process method of an implementing driver should *not* yield,
+  # unlike the process method of a Processor class.  Instead, it
+  # should treat its argument as an output of the dataflow and do
+  # something appropriate to the driver (write to file, database,
+  # terminal, &c.).
+  #
+  # Drivers are also responsible for implementing the lifecycle of
+  # processors and dataflows they drive.  A more complete version of
+  # the above driver class would:
+  #
+  #   * call the #setup_dataflow method when ready to trigger the
+  #     Processor#setup method on each processor in the dataflow
+  #
+  #   * call the #finalize_dataflow method when indicating that the
+  #     dataflow should consider a batch of records complete
+  #
+  #   * call the #finalize_and_stop_dataflow method to indicate the
+  #     last batch of records and to trigger the Processor#stop method
+  #     on each processor in the dataflow
+  #
+  # Driver instances are started by Runners which should delegate to
+  # the `start` method driver class itself.
+  # 
+  # @see Wukong::Local::StdioDriver for a complete example of a driver.
+  # @see Wukong::Local::Runner for an example of how runners call drivers.
+  module DriverMethods
+
+    attr_accessor :label
+    attr_accessor :settings
+    attr_accessor :dataflow
+
+    # Classes including DriverMethods should override this method with
+    # some way of handling the `output_record` that is appropriate for
+    # the driver.
+    #
+    # @param [Object] output_record
+    def process output_record
+      raise NotImplementedError.new("Define the #{self.class}#process method to handle output records from the dataflow")
+    end
+
+    # Construct a dataflow from the given `label` and `settings`.
+    #
+    # This method does **not** cause Processor#setup to be called on
+    # any of the processors in the dataflow.  Call the #setup_dataflow
+    # method to explicitly have setup occur.  This distinction is
+    # useful for drivers which themselves need to do complex
+    # initialization before letting processors in the dataflow
+    # initialize.
+    #
+    # @param [Symbol] label the name of the dataflow (or processor) to build
+    # @param [Hash] settings
+    # @param settings [String] :to Serialize all output via the named serializer (json, tsv)
+    # @param settings [String] :from Deserialize all input via the named deserializer (json, tsv)
+    # @param settings [String] :as Recordize each input as instances of the given class
+    # 
+    # @see #setup_dataflow
+    def construct_dataflow(label, settings={})
+      self.label    = label
+      self.settings = settings
+      prepend(:recordize)                       if settings[:as]
+      prepend("from_#{settings[:from]}".to_sym) if settings[:from]
+      append("to_#{settings[:to]}".to_sym)      if settings[:to]
+      build_dataflow
+    end
+
+    # Set up this driver.  Called before setting up any of the
+    # dataflow stages.
+    def setup
+    end
+
+    # Walks the dataflow and calls Processor#setup on each of the
+    # processors.
+    def setup_dataflow
+      setup
+      dataflow.each_stage do |stage|
+        stage.setup
+      end
+    end
+
+    # Send the given `record` through the dataflow.
+    #
+    # @param [Object] record
+    def send_through_dataflow(record)
+      wiring.start_with(dataflow.root).call(record)
+    end
+
+    # Perform finalization code for this driver.  Runs after #setup
+    # and before #stop.
+    def finalize
+    end
+
+    # Indicate a full batch of records has already been sent through
+    # and any batch-oriented or accumulative operations should trigger
+    # (e.g. - counting).
+    #
+    # Walks the dataflow calling Processor#finalize on each processor.
+    #
+    # On the *last* batch, the #finalize_and_stop_dataflow method
+    # should be called instead.
+    #
+    # @see #finalize_and_stop_dataflow
+    def finalize_dataflow
+      finalize
+      dataflow.each_stage do |stage|
+        stage.finalize(&wiring.advance(stage))
+      end
+    end
+
+    # Works similar to #finalize_dataflow but calls Processor#stop
+    # after calling Processor#finalize on each processor.
+    def finalize_and_stop_dataflow
+      finalize
+      dataflow.each_stage do |stage|
+        stage.finalize(&wiring.advance(stage))
+        stage.stop
+      end
+      stop
+    end
+
+    # Perform shutdown code for this driver.  Called after #finalize
+    # and after all stages have been finalized and stopped.
+    def stop
+    end
+
+    protected
+
+    # The builder for this driver's `label`, either for a Processor or
+    # a Dataflow.
+    #
+    # @return [Wukong::ProcessorBuilder, Wukong::DataflowBuilder]
+    def builder
+      return @builder if @builder
+      raise Wukong::Error.new("could not find definition for <#{label}>") unless Wukong.registry.registered?(label.to_sym)
+      @builder = Wukong.registry.retrieve(label.to_sym)
+    end
+
+    # Return the builder for this driver's dataflow.
+    #
+    # Even if a Processor was originally named by this driver's
+    # `label`, a DataflowBuilder will be returned here.  The
+    # DataflowBuilder is itself built from just the ProcessorBuilder
+    # alone.
+    #
+    # @return [Wukong::DataflowBuilder]
+    # @see #builder
+    def dataflow_builder
+      @dataflow_builder ||= (builder.is_a?(DataflowBuilder) ? builder : Wukong::DataflowBuilder.receive(for_class: Class.new(Wukong::Dataflow), stages: {label.to_sym => builder}))
+    end
+
+    # Build the dataflow using the #dataflow_builder and the supplied
+    # `settings`.
+    #
+    # @return [Wukong::Dataflow]
+    def build_dataflow
+      self.dataflow = dataflow_builder.build(settings)
+    end
+
+    # Add the processor with the given `new_label` in front of this
+    # driver's dataflow, making it into the new root of the dataflow.
+    #
+    # @param [Symbol] new_label
+    def prepend new_label
+      raise Wukong::Error.new("could not find processor <#{new_label}> to prepend") unless Wukong.registry.registered?(new_label)
+      dataflow_builder.prepend(Wukong.registry.retrieve(new_label))
+    end
+
+    # Add the processor with the given `new_label` at the end of each
+    # of this driver's dataflow's leaves.
+    #
+    # @param [Symbol] new_label
+    def append new_label
+      raise Wukong::Error.new("could not find processor <#{new_label}> to append") unless Wukong.registry.registered?(new_label)
+      dataflow_builder.append(Wukong.registry.retrieve(new_label))
+    end
+
+    # Returns the underlying Wiring object that will coordinate
+    # transfer of records from the driver to the dataflow and back to
+    # the driver.
+    #
+    # @return [Wiring]
+    def wiring
+      @wiring ||= Wiring.new(self, dataflow)
+    end
+    
+  end
+
+end

data/lib/wukong/driver/event_machine_driver.rb

@@ -0,0 +1,15 @@
+module Wukong
+  module EventMachineDriver
+    include DriverMethods
+
+    def self.included klass
+      klass.class_eval do
+        def self.add_signal_traps
+          Signal.trap('INT')  { log.info 'Received SIGINT. Stopping.'  ; EM.stop }
+          Signal.trap('TERM') { log.info 'Received SIGTERM. Stopping.' ; EM.stop }                  
+        end
+      end
+    end
+      
+  end
+end

data/lib/wukong/driver/wiring.rb

@@ -0,0 +1,68 @@
+module Wukong
+
+  # Provides a very Ruby-minded way of walking a dataflow connected to
+  # a driver.
+  class Wiring
+
+    # The driver instance that likely calls the #start_with method and
+    # provides a #process method to be called by this wiring.
+    attr_accessor :driver
+
+    # The dataflow being wired.
+    attr_accessor :dataflow
+
+    # Construct a new Wiring for the given `driver` and `dataflow`.
+    #
+    # @param [#process] driver
+    # @param [Wukong::Dataflow] dataflow
+    def initialize(driver, dataflow)
+      @driver    = driver
+      @dataflow  = dataflow
+    end
+
+    # Return a proc which, if called with a record, will process that
+    # record through each of the given `stages` as well as through the
+    # rest of the dataflow ahead of them.
+    #
+    # @param [Array<Wukong::Stage>] stages
+    # @return [Proc]
+    def start_with(*stages)
+      to_proc.curry.call(stages)
+    end
+
+    # Return a proc (the output of #start_with) which will process
+    # records through the stages that are ahead of the given stage.
+    #
+    # @param [Wukong::Stage] stage
+    # @return [Proc]
+    #
+    # @see #start_with
+    def advance(stage)
+      # This is where the tree of procs will terminate, but only after
+      # having passed all output records through the driver -- the
+      # last "stage".
+      return start_with() if stage.nil? || stage == driver
+
+      # Otherwise we're still in the middle of the tree...
+      descendents = dataflow.descendents(stage)
+      if descendents.empty?
+        # No descendents it means we've reached a leaf of the tree so
+        # we'll run records through the driver to generate output.
+        start_with(driver)
+      else
+        # Otherwise continue down the tree of procs...
+        start_with(*descendents)
+      end
+    end
+    
+    # :nodoc:
+    def to_proc
+      return @wiring if @wiring
+      @wiring = Proc.new do |stages, record|
+        stages.each do |stage|
+          stage.process(record, &advance(stage)) if stage
+        end
+      end
+    end
+  end
+end

data/lib/wukong/local.rb

@@ -0,0 +1,42 @@
+module Wukong
+
+  # Provides methods for supporting the running of Wukong processors
+  # and dataflows entirely locally, without any frameworks like Hadoop
+  # or Storm.
+  #
+  # This module is actually a plugin for Wukong.
+  module Local
+    include Plugin
+
+    # Configures the given +settings+ object with all settings
+    # specific to Wukong::Local for the given program +name+.
+    #
+    # @param [Configliere::Param] settings the settings to configure
+    # @param [String] program the name of the currently executing program
+    def self.configure settings, program
+      case program
+      when 'wu-local'
+        settings.define :run,  description: "Name of the processor or dataflow to use. Defaults to basename of first argument", flag: 'r'
+        
+        settings.define :from, description: "Parse input from given data format (json, tsv, &c.) before processing"
+        settings.define :to,   description: "Convert input to given data format (json, tsv, &c.) before emitting"
+        settings.define :as,   description: "Call Class.receive on each input (will run after --from)", type: Class
+      when 'wu-source'
+        settings.define :per_sec,    description: "Number of events produced per second", type: Float
+        settings.define :period,     description: "Number of seconds between events (overrides --per_sec)", type: Float
+        settings.define :batch_size, description: "Trigger a finalize across the dataflow each time this many records are processed", type: Integer
+      end
+    end
+
+    # Boots Wukong::Local using the given +settings+ at the given
+    # +root.
+    #
+    # @param [Configliere::Param] settings the settings to use to boot
+    # @param [String] root the root directory to boot in
+    def self.boot(settings, root)
+    end
+    
+  end
+end
+
+require_relative('local/runner')

data/lib/wukong/local/runner.rb

@@ -0,0 +1,96 @@
+require_relative 'stdio_driver'
+
+module Wukong
+  module Local
+
+    # Implements the Runner for wu-local.
+    class LocalRunner < Wukong::Runner
+
+      include Wukong::Logging
+
+      usage "PROCESSOR|FLOW"
+      
+      description <<-EOF.gsub(/^ {8}/, '')
+        wu-local is a tool for running Wukong processors and flows locally on
+        the command-line.  Use wu-local by passing it a processor and feeding
+        in some data:
+
+          $ echo 'UNIX is Clever and Fun...' | wu-local tokenizer.rb
+          UNIX
+          is
+          Clever
+          and
+          Fun
+
+        If your processors have named fields you can pass them in as
+        arguments:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4
+          UNIX
+          Clever
+
+        You can chain processors and calls to wu-local together:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4 | wu-local downcaser.rb
+          unix
+          clever
+
+        Which is a good way to develop a combined data flow which you can
+        again test locally:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenize_and_downcase_big_words.rb
+          unix
+          clever
+      EOF
+
+      # Returns the name of the dataflow we're going to run.
+      #
+      # @return [String]
+      def dataflow
+        arg      = args.first
+        basename = File.basename(arg.to_s, '.rb')
+
+        case
+        when settings[:run]          then settings[:run]
+        when arg && File.exist?(arg) then basename
+        else arg
+        end
+      end
+      alias_method :processor, :dataflow
+
+      # Validates the chosen processor.
+      #
+      # @raise [Wukong::Error] if it finds a problem
+      # @return [true]
+      def validate
+        raise Error.new("Must provide a processor or dataflow to run, via either the --run option or as the first argument") if dataflow.nil? || dataflow.empty?
+        raise Error.new("No such processor or dataflow <#{dataflow}>") unless registered?(dataflow)
+        true
+      end
+
+      # Adds a customized help message built from the Processor
+      # # itself.
+      def setup
+        super()
+        dataflow_class_for(dataflow).configure(settings) if registered?(dataflow)
+      end
+
+      # Starts up the driver with the right dataflow and settings.
+      #
+      # Starts the EventMachine reactor before starting the driver.
+      def run
+        EM.run do
+          driver.start(dataflow, settings)
+        end
+      end
+
+      # The class used 
+      #
+      # @return [Class, #start]
+      def driver
+        StdioDriver
+      end
+      
+    end
+  end
+end