wukong 3.0.0.pre3 → 3.0.0

data/lib/wukong/local/event_machine_driver.rb

@@ -0,0 +1,27 @@
+module Wukong
+
+  # A module which can be included by other drivers which lets them
+  # use EventMachine under the hood.
+  module EventMachineDriver
+    
+    include DriverMethods
+
+    # :nodoc:
+    def self.included klass
+      klass.class_eval do
+        def self.add_signal_traps
+          Signal.trap('INT')  { log.info 'Received SIGINT. Stopping.'  ; EM.stop }
+          Signal.trap('TERM') { log.info 'Received SIGTERM. Stopping.' ; EM.stop }                  
+        end
+      end
+    end
+
+    # :nodoc:
+    def initialize(label, settings)
+      super
+      @settings = settings      
+      @dataflow = construct_dataflow(label, settings)
+    end
+    
+  end
+end

data/lib/wukong/local/runner.rb

@@ -0,0 +1,98 @@
+require_relative 'stdio_driver'
+require_relative 'tcp_driver'
+
+module Wukong
+  module Local
+
+    # Implements the Runner for wu-local.
+    class LocalRunner < Wukong::Runner
+
+      include Wukong::Logging
+
+      usage "PROCESSOR|FLOW"
+      
+      description <<-EOF.gsub(/^ {8}/, '')
+        wu-local is a tool for running Wukong processors and flows locally on
+        the command-line.  Use wu-local by passing it a processor and feeding
+        in some data:
+
+          $ echo 'UNIX is Clever and Fun...' | wu-local tokenizer.rb
+          UNIX
+          is
+          Clever
+          and
+          Fun
+
+        If your processors have named fields you can pass them in as
+        arguments:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4
+          UNIX
+          Clever
+
+        You can chain processors and calls to wu-local together:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4 | wu-local downcaser.rb
+          unix
+          clever
+
+        Which is a good way to develop a combined data flow which you can
+        again test locally:
+
+          $ echo 'UNIX is clever and fun...' | wu-local tokenize_and_downcase_big_words.rb
+          unix
+          clever
+      EOF
+
+      # Returns the name of the processor we're going to run.
+      #
+      # @return [String]
+      def processor
+        arg      = args.first
+        basename = File.basename(arg.to_s, '.rb')
+
+        case
+        when settings[:run]          then settings[:run]
+        when arg && File.exist?(arg) then basename
+        else arg
+        end
+      end
+
+      # Validates the chosen processor.
+      #
+      # @raise [Wukong::Error] if it finds a problem
+      # @return [true]
+      def validate
+        raise Error.new("Must provide a processor or dataflow to run, via either the --run option or as the first argument") if processor.nil? || processor.empty?
+        raise Error.new("No such processor or dataflow <#{processor}>") unless registered?(processor)
+        true
+      end
+
+      # Adds a customized help message built from the Processor
+      # # itself.
+      def setup
+        super()
+        dataflow_class_for(processor).configure(settings) if processor?(processor)
+      end
+
+      # Runs either the StdioDriver or the TCPDriver, depending on
+      # what settings were passed.
+      def run
+        EM.run do 
+          driver.start(processor, settings)
+        end
+      end
+
+      # The driver this Runner will use.
+      #
+      # Defaults to the Wukong::Local::StdioDriver, but will use the
+      # TcpDriver if it has a :port setting defined.
+      #
+      # @return [Wukong::Local::TCPDriver, Wukong::Local::StdioDriver]
+      def driver
+        (settings[:tcp_port] ? TCPDriver : StdioDriver)
+      end
+      
+    end
+  end
+end

data/lib/wukong/local/stdio_driver.rb

@@ -0,0 +1,44 @@
+require_relative('event_machine_driver')
+module Wukong
+  module Local
+
+    # A class for driving processors over the STDIN/STDOUT protocol.
+    class StdioDriver < EM::P::LineAndTextProtocol
+      include EventMachineDriver
+      include Processor::StdoutProcessor
+      include Logging
+      
+      def self.start(label, settings = {})
+        EM.attach($stdin, self, label, settings)
+      end
+
+      def post_init      
+        self.class.add_signal_traps
+        setup_dataflow
+      end
+
+      def receive_line line
+        driver.send_through_dataflow(line)
+      rescue => e
+        error = Wukong::Error.new(e)
+        EM.stop
+        
+        # We'd to *raise* `error` here and have it be handled by
+        # Wukong::Runner.run but we are fighting with EventMachine.
+        # It seems no matter what we do, EventMachine will swallow any
+        # Exception raised here (including SystemExit) and exit the
+        # Ruby process with a return code of 0.
+        #
+        # Instead we just log the message that *would* have gotten
+        # logged by Wukong::Runner.run and leave it to EventMachine to
+        # exit very unnaturally.
+        log.error(error.message)
+      end
+
+      def unbind
+        finalize_and_stop_dataflow
+        EM.stop
+      end
+    end
+  end
+end

data/lib/wukong/local/tcp_driver.rb

@@ -0,0 +1,47 @@
+require_relative('event_machine_driver')
+module Wukong
+  module Local
+    
+    # A class for driving processors over a TCP protocol.
+    class TCPDriver < EM::P::LineAndTextProtocol
+      include EventMachineDriver
+      include Processor::BufferedProcessor
+      include Logging
+
+      def self.start(label, settings = {})
+        host = (settings[:host] || Socket.gethostname) rescue 'localhost'
+        port = (settings[:port] || 9000).to_i          rescue 9000
+        EM.start_server(host, port, self, label, settings)
+        log.info "Server started on #{host} on port #{port}"
+        add_signal_traps
+      end
+
+      def post_init
+        port, ip = Socket.unpack_sockaddr_in(get_peername)
+        log.info "Connected to #{ip} on #{port}"
+        setup_dataflow
+      end
+
+      def receive_line line
+        @buffer = []      
+        operation = proc { driver.send_through_dataflow(line) }
+        callback  = proc { flush_buffer @buffer }
+        EM.defer(operation, callback)
+      rescue => e
+        EM.stop
+        raise Wukong::Error.new(e)
+      end
+
+      def flush_buffer records
+        send_data(records.join("\n") + "\n")
+        records.clear
+      end
+
+      def unbind
+        finalize_and_stop_dataflow
+        EM.stop
+      end
+      
+    end
+  end
+end

data/lib/wukong/logger.rb

@@ -1,10 +1,19 @@
-module Wukong  
+module Wukong
+
   class LogFactory
 
     attr_reader :created_log
 
-    def self.defaults
-      Log4r::StderrOutputter.new('console', formatter: Log4r::PatternFormatter.new(pattern: "%l %d [%-20c] -- %m"))
+    def self.default_outputter klass
+      Log4r::StderrOutputter.new('console', formatter: default_formatter(klass))
+    end
+
+    def self.default_formatter klass
+      Log4r::PatternFormatter.new(pattern: default_pattern(klass))
+    end
+
+    def self.default_pattern klass
+      "%l %d [%-20c] -- %m"
     end
 
     def self.configure(klass, options = {})
@@ -14,7 +23,7 @@ module Wukong
 
     def initialize(logger, config)
       @created_log = logger.is_a?(Log4r::Logger) ? logger : Log4r::Logger.new(logger.to_s)
-      outputter(LogFactory.defaults) unless ancestry_has_outputter?(@created_log)
+      outputter(LogFactory.default_outputter(logger)) unless ancestry_has_outputter?(@created_log)
       apply_options(config)
     end
 
@@ -33,7 +42,7 @@ module Wukong
         begin
           send(option, value)
         rescue
-          raise "invalid log option"
+          raise Error.new("Error setting option <#{option}> to value <#{value}>")
         end
       end
     end
@@ -51,7 +60,7 @@ module Wukong
         debug: Log4r::DEBUG,
         info:  Log4r::INFO,
         warn:  Log4r::WARN
-      }.fetch(lvl){ raise "invalid log level" }
+      }.fetch(lvl){ raise Error.new("Invalid log level: <#{lvl}>") }
     end
 
     def pattern ptrn
@@ -75,7 +84,7 @@ module Wukong
     def self.included klass
       if klass.ancestors.include?(Gorillib::Model)
         klass.class_eval do
-          field(:log, Whatever, :default => ->{ Wukong::LogFactory.configure(self.class) }) 
+          field(:log, Whatever, :default => ->{ Wukong::LogFactory.configure(self.class) }, :doc => "Shared Wukong logger")
 
           def receive_log params
             @log = LogFactory.configure(self.class, params)

data/lib/wukong/plugin.rb

@@ -0,0 +1,48 @@
+module Wukong
+
+  # An array of known plugins.
+  PLUGINS = []
+
+  # Asks each loaded plugin to configure the given +settings+ for the
+  # given +program_name+.
+  #
+  # @param [Configliere::Param] settings the settings to be configured by each plugin
+  # @param [String] program_name the name of the currently executing program
+  def self.configure_plugins(settings, program_name)
+    PLUGINS.each do |plugin|
+      plugin.configure(settings, program_name) if plugin.respond_to?(:configure)
+    end
+  end
+
+  # Asks each loaded plugin to boot itself from the given +settings+
+  # in the given +root+ directory.
+  #
+  # @param [Configliere::Param] settings the settings for each plugin to boot from
+  # @param [String] root the root directory the plugins are booting in
+  def self.boot_plugins(settings, root)
+    PLUGINS.each do |plugin|
+      plugin.boot(settings, root) if plugin.respond_to?(:boot)
+    end
+  end
+
+  # Include this module in your own class or module to have it
+  # register itself as a Wukong plugin.
+  #
+  # Your class or module must define the following methods:
+  #
+  # * `configure` called with a (pre-resolved) Configliere::Param argument and the basename of the running program
+  # * `boot` called with a (resolved) Configliere::Param argument and the current working directory of the running program, reacts to any settings as necessary
+  #
+  # Subclasses of Wukong::Runner will automatically load and boot each
+  # plugin.
+  module Plugin
+    # :nodoc:
+    def self.included mod
+      PLUGINS << mod unless PLUGINS.include?(mod)
+    end
+  end
+  
+end
+
+    
+    

data/lib/wukong/processor.rb

@@ -15,15 +15,12 @@ module Wukong
     include Logging
     include Vayacondios::Notifications
     
-    field :action,   Whatever
+    field :action, Whatever, :doc => false
 
     class << self
 
-      def describe desc
-        @description = desc
-      end
-      
-      def description
+      def description desc=nil
+        @description = desc if desc
         @description
       end
       
@@ -47,6 +44,29 @@ module Wukong
         instance_variable_set("@serialization_#{direction}", label) if %w[ tsv json xml ].include?(label.to_s)
       end
 
+      def configure(settings)
+        settings.description = description if description
+        fields.each_pair do |name, field|
+          next if field.doc == false || field.doc.to_s == 'false'
+          next if [:log, :notifier].include?(name)
+          field_props = {}.tap do |props|
+            props[:description] = field.doc unless field.doc == "#{name} field"
+            field_type = (field.type.respond_to?(:product) ? field.type.product : field.type)
+            configliere_type = case field_type
+            when String                then nil
+            when TrueClass, FalseClass then :boolean
+            else field_type
+            end
+            
+            props[:type]        = configliere_type if configliere_type
+            props[:default]     = field.default if field.default
+          end
+          existing_value = settings[name]
+          settings.define(name, field_props)
+          settings[name] = existing_value unless existing_value.nil?
+        end
+      end
+
     end
         
     def expected_record_type(type)
@@ -57,21 +77,38 @@ module Wukong
       self.class.instance_variable_get("@serialization_#{direction.to_s}")
     end
     
-    # This is a placeholder method intended to be overridden
-    def perform_action(*args) ; end 
+    # When instantiated with a block, the block will replace this
+    # method.
+    #
+    # @param [Array<Object>] args
+    # @yield record a record that might be yielded by the block
+    # @yieldparam [Object] record the yielded record
+    def perform_action(*args)
+    end 
     
-    # The action attribute is turned into the perform action method
+    # :nodoc:
+    #
+    # The action attribute is turned into the perform action method.
+    #
+    # @param [Proc] action
     def receive_action(action)
       self.define_singleton_method(:perform_action, &action)
     end
 
-    # This method is called after the processor class has been instantiated
-    # but before any records are given to it to process
+    # This method is called after the processor class has been
+    # instantiated but before any records are given to it to process.
+    #
+    # Override this method in your subclass.
     def setup
     end
 
-    # This method is called once per record
-    # Override this in your subclass
+    # This method is called once per record.
+    #
+    # Override this method in your subclass.
+    #
+    # @param [Object] record
+    # @yield record the record you want to yield
+    # @yieldparam [Object] record the yielded record
     def process(record, &emit)
       yield record
     end
@@ -83,13 +120,18 @@ module Wukong
     # This can be used within an aggregating processor (like a reducer
     # in a map/reduce job) to start processing the final aggregate of
     # records since the "last record" has already been received.
+    #
+    # Override this method in your subclass
+    #
+    # @yield record the record you want to yield
+    # @yieldparam [Object] record the yielded record
     def finalize
     end
 
     # This method is called after all records have been passed.  It
     # signals that processing should stop.
-      
-    # This method is called after all records have been processed
+    #
+    # Override this method in your subclass.
     def stop
     end
 

data/lib/wukong/rake_helper.rb

@@ -0,0 +1,6 @@
+require 'rake'
+require 'wukong'
+
+task :environment => [] do
+  Wukong::Runner.run
+end