wukong 3.0.1 → 4.0.0

data/lib/wukong/local.rb

@@ -16,13 +16,15 @@ module Wukong
     def self.configure settings, program
       case program
       when 'wu-local'
-        settings.define :run,      description: "Name of the processor or dataflow to use. Defaults to basename of the given path.", flag: 'r'
-        settings.define :tcp_port, description: "Consume TCP requests on the given port instead of lines over STDIN", type: Integer, flag: 't'
+        settings.define :run,  description: "Name of the processor or dataflow to use. Defaults to basename of first argument", flag: 'r'
         
         settings.define :from, description: "Parse input from given data format (json, tsv, &c.) before processing"
         settings.define :to,   description: "Convert input to given data format (json, tsv, &c.) before emitting"
-
-        settings.define :consumes, description: "Parse input as instances of given model class before processing", type: Class
+        settings.define :as,   description: "Call Class.receive on each input (will run after --from)", type: Class
+      when 'wu-source'
+        settings.define :per_sec,    description: "Number of events produced per second", type: Float
+        settings.define :period,     description: "Number of seconds between events (overrides --per_sec)", type: Float
+        settings.define :batch_size, description: "Trigger a finalize across the dataflow each time this many records are processed", type: Integer
       end
     end
 

data/lib/wukong/local/runner.rb

@@ -1,5 +1,4 @@
 require_relative 'stdio_driver'
-require_relative 'tcp_driver'
 
 module Wukong
   module Local
@@ -44,10 +43,10 @@ module Wukong
           clever
       EOF
 
-      # Returns the name of the processor we're going to run.
+      # Returns the name of the dataflow we're going to run.
       #
       # @return [String]
-      def processor
+      def dataflow
         arg      = args.first
         basename = File.basename(arg.to_s, '.rb')
 
@@ -57,14 +56,15 @@ module Wukong
         else arg
         end
       end
+      alias_method :processor, :dataflow
 
       # Validates the chosen processor.
       #
       # @raise [Wukong::Error] if it finds a problem
       # @return [true]
       def validate
-        raise Error.new("Must provide a processor or dataflow to run, via either the --run option or as the first argument") if processor.nil? || processor.empty?
-        raise Error.new("No such processor or dataflow <#{processor}>") unless registered?(processor)
+        raise Error.new("Must provide a processor or dataflow to run, via either the --run option or as the first argument") if dataflow.nil? || dataflow.empty?
+        raise Error.new("No such processor or dataflow <#{dataflow}>") unless registered?(dataflow)
         true
       end
 
@@ -72,25 +72,23 @@ module Wukong
       # # itself.
       def setup
         super()
-        dataflow_class_for(processor).configure(settings) if processor?(processor)
+        dataflow_class_for(dataflow).configure(settings) if registered?(dataflow)
       end
 
-      # Runs either the StdioDriver or the TCPDriver, depending on
-      # what settings were passed.
+      # Starts up the driver with the right dataflow and settings.
+      #
+      # Starts the EventMachine reactor before starting the driver.
       def run
-        EM.run do 
-          driver.start(processor, settings)
+        EM.run do
+          driver.start(dataflow, settings)
         end
       end
 
-      # The driver this Runner will use.
-      #
-      # Defaults to the Wukong::Local::StdioDriver, but will use the
-      # TcpDriver if it has a :port setting defined.
+      # The class used 
       #
-      # @return [Wukong::Local::TCPDriver, Wukong::Local::StdioDriver]
+      # @return [Class, #start]
       def driver
-        (settings[:tcp_port] ? TCPDriver : StdioDriver)
+        StdioDriver
       end
       
     end

data/lib/wukong/local/stdio_driver.rb

@@ -1,33 +1,74 @@
-require_relative('event_machine_driver')
 module Wukong
   module Local
 
     # A class for driving processors over the STDIN/STDOUT protocol.
+    #
+    # Relies on EventMachine's [LineAndTextProtocol](http://eventmachine.rubyforge.org/EventMachine/Protocols/LineText2.html).
     class StdioDriver < EM::P::LineAndTextProtocol
-      include EventMachineDriver
-      include Processor::StdoutProcessor
-      include Logging
       
+      include DriverMethods
+      include Logging
+
+      #
+      # == Startup == 
+      #
+
+      # Start a new StdioDriver.
+      #
+      # @param [Symbol] the name of the processor or dataflow to drive
+      # @param [Configliere::Param] settings the settings to use
       def self.start(label, settings = {})
         EM.attach($stdin, self, label, settings)
       end
 
+      # :nodoc:
+      def initialize(label, settings)
+        super
+        construct_dataflow(label, settings)
+      end
+
+      # Ensures that $stdout is synced.
+      def setup()
+        $stdout.sync
+      end
+
+      # Adds signal traps for SIGINT and SIGTERM to Ensure we capture
+      # C-c and friends, stop the EventMachine reactor, &c.
+      def self.add_signal_traps
+        Signal.trap('INT')  { log.info 'Received SIGINT. Stopping.'  ; EM.stop }
+        Signal.trap('TERM') { log.info 'Received SIGTERM. Stopping.' ; EM.stop }
+      end
+
+      # Called by EventMachine framework after successfully attaching
+      # to $stdin.
+      #
+      # Adds signal handlers and calls the #setup_dataflow method.
       def post_init      
         self.class.add_signal_traps
         setup_dataflow
       end
-
+      
+      #
+      # == Reading Input == 
+      #
+      
+      # Called by EventMachine framework after successfully reading a
+      # line from $stdin.
+      #
+      # @param [String] line
       def receive_line line
-        driver.send_through_dataflow(line)
+        send_through_dataflow(line)
       rescue => e
         error = Wukong::Error.new(e)
-        EM.stop
+        # EM.stop
         
-        # We'd to *raise* `error` here and have it be handled by
-        # Wukong::Runner.run but we are fighting with EventMachine.
-        # It seems no matter what we do, EventMachine will swallow any
-        # Exception raised here (including SystemExit) and exit the
-        # Ruby process with a return code of 0.
+        # We'd like to *raise* `error` here and have it be handled by
+        # Wukong::Runner.run but we are fighting with EventMachine.run
+        # which executes in the middle.
+        #
+        # It seems no matter what we do, EventMachine.run will swallow
+        # any Exception raised here (including SystemExit) and exit
+        # the Ruby process with a return code of 0.
         #
         # Instead we just log the message that *would* have gotten
         # logged by Wukong::Runner.run and leave it to EventMachine to
@@ -35,6 +76,25 @@ module Wukong
         log.error(error.message)
       end
 
+      #
+      # == Handling Output == 
+      #
+      
+      # Writes a record to $stdout.
+      #
+      # @param [#to_s] record
+      def process(record)
+        $stdout.puts record
+      end
+
+      #
+      # == Shutdown == 
+      #
+
+      # Called by EventMachine framework after EOF from $stdin.
+      #
+      # Calls #finalize_and_stop_dataflow method and stops the
+      # EventMachine reactor.
       def unbind
         finalize_and_stop_dataflow
         EM.stop

data/lib/wukong/processor.rb

@@ -13,7 +13,6 @@ module Wukong
   # local machine.  You can glue processors together
   class Processor < Hanuman::Stage
     include Logging
-    include Vayacondios::Notifications
     
     field :action, Whatever, :doc => false
 
@@ -23,32 +22,12 @@ module Wukong
         @description = desc if desc
         @description
       end
-      
-      def consumes(*args)
-        options   = args.extract_options!
-        @consumes = options[:as]
-        validate_and_set_serialization(:from, args.first)
-      end
-
-      def produces(*args)
-        options   = args.extract_options!
-        @produces = options[:as]
-        validate_and_set_serialization(:to, args.first)      
-      end
             
-      def valid_serializer? label
-        label
-      end
-
-      def validate_and_set_serialization(direction, label)
-        instance_variable_set("@serialization_#{direction}", label) if %w[ tsv json xml ].include?(label.to_s)
-      end
-
       def configure(settings)
         settings.description = description if description
         fields.each_pair do |name, field|
           next if field.doc == false || field.doc.to_s == 'false'
-          next if [:log, :notifier].include?(name)
+          next if [:log].include?(name)
           field_props = {}.tap do |props|
             props[:description] = field.doc unless field.doc == "#{name} field"
             field_type = (field.type.respond_to?(:product) ? field.type.product : field.type)
@@ -69,14 +48,6 @@ module Wukong
 
     end
         
-    def expected_record_type(type)
-      self.class.instance_variable_get("@#{type}")
-    end
-    
-    def expected_serialization(direction)
-      self.class.instance_variable_get("@serialization_#{direction.to_s}")
-    end
-    
     # When instantiated with a block, the block will replace this
     # method.
     #

data/lib/wukong/runner.rb

@@ -1,6 +1,7 @@
 require_relative("runner/code_loader")
 require_relative("runner/deploy_pack_loader")
 require_relative("runner/boot_sequence")
+require_relative("runner/command_runner")
 
 module Wukong
 
@@ -18,6 +19,7 @@ module Wukong
     include CodeLoader
     include DeployPackLoader
     include BootSequence
+    include CommandRunner
 
     # The settings object that will be configured and booted from.
     # All plugins will configure this object.

data/lib/wukong/runner/command_runner.rb

@@ -0,0 +1,44 @@
+module Wukong
+  class Runner
+
+    # Provides methods for executing commandlines.
+    module CommandRunner
+
+      private
+
+      # Execute a command composed of the given parts.
+      #
+      # Will print the command instead if the <tt>--dry_run</tt>
+      # option was given.
+      #
+      # Will *not* raise an error if the command fails.
+      #
+      # @param [Array<String>] argv
+      def execute_command(*argv)
+        command = argv.flatten.reject(&:blank?).join(" \\\n    ")
+        if settings[:dry_run]
+          log.info("Dry run:")
+          puts command
+        else
+          output = `#{command}`
+          puts output unless output.empty?
+        end
+      end
+
+      # Execute a command composed of the given parts.
+      #
+      # Will print the command instead if the <tt>--dry_run</tt>
+      # option was given.
+      #
+      # *Will* raise an error if the command fails.
+      #
+      # @param [Array<String>] argv
+      def execute_command!(*argv)
+        execute_command(argv)
+        raise Error.new("Command failed!") unless $?.success?
+      end
+      
+    end
+
+  end
+end

data/lib/wukong/source.rb

@@ -0,0 +1,33 @@
+module Wukong
+
+  # Provides a runner for periodically triggering a dataflow or
+  # processor.
+  module Source
+    include Plugin
+
+    # Configures the given +settings+ object with all settings
+    # specific to Wukong::Source for the given program +name+.
+    #
+    # @param [Configliere::Param] settings the settings to configure
+    # @param [String] program the name of the currently executing program
+    def self.configure settings, program
+      case program
+      when 'wu-source'
+        settings.define :per_sec,    description: "Number of events produced per second", type: Float
+        settings.define :period,     description: "Number of seconds between events (overrides --per_sec)", type: Float
+        settings.define :batch_size, description: "Trigger a finalize across the dataflow each time this many records are processed", type: Integer
+      end
+    end
+
+    # Boots Wukong::Source using the given +settings+ at the given
+    # +root.
+    #
+    # @param [Configliere::Param] settings the settings to use to boot
+    # @param [String] root the root directory to boot in
+    def self.boot(settings, root)
+    end
+    
+  end
+end
+
+require_relative('source/source_runner')

data/lib/wukong/source/source_driver.rb

@@ -0,0 +1,74 @@
+module Wukong
+  module Source
+
+    # A driver which works just like the `Wukong::Local::StdioDriver`
+    # except it ignores input from `STDIN` and instead generates its
+    # own input records according to some periodic schedule.  Each
+    # consecutive record produced will be an incrementing positive
+    # integer (as a string), starting with '1'.
+    class SourceDriver < Wukong::Local::StdioDriver
+      
+      include Logging
+
+      # The index of the record.
+      attr_accessor :index
+
+      # The number of records after which a `Processor#finalize` will
+      # be called.
+      attr_accessor :batch_size
+
+      # Sets the initial value of `index` to 1 and sets the batch size
+      # (only if it's positive).
+      def post_init
+        super()
+        self.index = 1
+        self.batch_size = settings[:batch_size].to_i if settings[:batch_size] && settings[:batch_size].to_i > 0
+      end
+
+      # Starts periodically feeding the processor or dataflow given by
+      # `label` using the given `settings`.
+      #
+      # @param [String, Symbol] label
+      # @param [Configliere::Param, Hash] settings
+      def self.start(label, settings={})
+        driver = new(:foobar, label, settings) # i don't think the 1st argument matters here...
+        driver.post_init
+
+        period = case
+        when settings[:period]  then settings[:period]
+        when settings[:per_sec] then (1.0 / settings[:per_sec]) rescue 1.0
+        else 1.0
+        end
+        driver.create_event
+        EventMachine::PeriodicTimer.new(period) { driver.create_event }
+      end
+
+      # Creates a new event using the following steps:
+      #
+      # 1. Feeds a record with the existing `index` to the dataflow.
+      # 2. Increments the `index`.
+      # 3. Finalizes the dataflow if the number of records is a
+      #    multiple of the `batch_size`.
+      #
+      # @see DriverMethods
+      def create_event
+        receive_line(index.to_s)
+        self.index += 1
+        finalize_dataflow if self.batch_size && (self.index % self.batch_size) == 0
+      end
+
+      # Outputs a `record` from the dataflow or processor to `STDOUT`.
+      #
+      # `STDOUT` will automatically be flushed to force output to
+      # prevent the feeling of "no output" when the looping period is
+      # long.
+      #
+      # @param [Object] record the record yielded by the processor or the terminal node(s) of the dataflow
+      def process record
+        $stdout.puts record
+        $stdout.flush
+      end
+      
+    end
+  end
+end

data/lib/wukong/source/source_runner.rb

@@ -0,0 +1,38 @@
+require_relative('source_driver')
+module Wukong
+  module Source
+
+    # Implements the `wu-source` command.
+    class SourceRunner < Wukong::Local::LocalRunner
+
+      usage "PROCESSOR|DATAFLOW"
+
+      description <<-EOF.gsub(/^ {8}/,'')
+        wu-source is a tool for using Wukong processors as sources of
+        data in streams.
+
+        Run any Wukong processor as a source for data:
+
+          $ wu-source fake_log_data
+          205.4.75.208 - 3918471017 [27/Nov/2012:05:06:57 -0600] "GET /products/eget HTTP/1.0" 200 25600
+          63.181.105.15 - 3650805763 [27/Nov/2012:05:06:57 -0600] "GET /products/lacinia-nulla-vitae HTTP/1.0" 200 3790
+          227.190.78.101 - 39543891 [27/Nov/2012:05:06:58 -0600] "GET /products/odio-nulla-nulla-ipsum HTTP/1.0" 200 31718
+          ...
+
+        The fake_log_data processor will receive an event once every
+        second.  Each event will consist of a single string giving a
+        consecutive integer starting with '1' as the first event.
+      EOF
+      
+      include Logging
+
+      # The driver class used by `wu-source`.
+      #
+      # @return [Class]
+      def driver
+        SourceDriver
+      end
+
+    end
+  end
+end