ul-wukong 4.1.0

data/lib/wukong/runner/boot_sequence.rb

@@ -0,0 +1,123 @@
+require_relative('help_message')
+module Wukong
+  class Runner
+
+    # The boot sequence of a runner consists of the following phases,
+    # each corresponding to a method provided by this module.
+    #
+    # * #load -- loads all application code
+    # * #configure -- configures settings from core Wukong, any loaded plugins, and any application code
+    # * #resolve -- resolves settings
+    # * #setup -- boots core Wukong and all loaded plugins
+    # * #validate -- validates command-line args
+    # * #run -- starts the runner running
+    #
+    # Each method can be separately overriden, allowing for a lot of
+    # customizability for different kinds of runners.
+    module BootSequence
+
+      include HelpMessage
+      
+      # Boot this Runner, calling in order:
+      #
+      # * #load
+      # * #configure
+      # * #resolve
+      # * #setup
+      # * #validate
+      # * #run or #die
+      #
+      # If `override_settings` is passed then merge it over the
+      # Runner's usual settings (this is useful for unit tests where
+      # settings are injected in ways different from the usual
+      # workflow).
+      #
+      # @param [Configliere::Param] override_settings
+      def boot!(override_settings=nil)
+        load
+        configure
+        resolve
+        setup
+        settings.merge!(override_settings) if override_settings
+        
+        case
+        when help_given? then dump_help_and_exit!
+        when validate    then run
+        else
+          die("Invalid arguments")
+        end
+      end
+
+      private
+
+      # Loads all code necessary for this Runner to perform, including:
+      #
+      # * any code associated with being inside of a deploy pack
+      # * any code passed in as (unknown rest) arguments on the command-line
+      def load
+        load_deploy_pack
+        load_args
+      end
+
+      # Endows the settings with everything it needs, including usage,
+      # description, and any define's provided by this Runner class or
+      # any plugins.
+      def configure
+        settings.use(:commandline)
+        settings.description = description if description
+        u = usage
+        settings.define_singleton_method(:usage){ u } if u
+        Wukong.configure_plugins(settings, program_name)
+      end
+
+      # Resolves the settings.
+      #
+      # Rescues some of the annoying RuntimeErrors thrown by
+      # Configliere...
+      def resolve
+        begin
+          strip_help_param!
+          settings.resolve!
+          true
+        rescue RuntimeError, SystemExit => e
+          raise Error.new(e)
+        end
+      end
+      
+      # Performs any setup code necessary before run.
+      #
+      # Boots all plugins by default.  If you override this code, make
+      # sure to either call `super` or boot plugins yourself.
+      def setup
+        Wukong.boot_plugins(settings, root)
+      end
+
+      # Validates the command-line args.  Raise a Wukong::Error in
+      # this method to terminate execution with a specific or custom
+      # error.
+      #
+      # Return false-like to terminate with a generic argument error.
+      #
+      # @return [true, false]
+      def validate
+        true
+      end
+      
+      # Run this runner.
+      #
+      # You'll want to override this method in your own Runner class.
+      def run
+      end
+
+      # Kill this runner with the given error `message` and exit
+      # `code`.
+      #
+      # @param [String] message
+      # @param [Integer] code
+      def die message=nil, code=126
+        self.class.die(message, code)
+      end
+
+    end
+  end
+end

data/lib/wukong/runner/code_loader.rb

@@ -0,0 +1,52 @@
+module Wukong
+  class Runner
+
+    # Defines methods to help a Runner class load code passed in
+    # dynamically on the command-line.
+    #
+    # The default behavior of code in this module is to load any Ruby
+    # files (ending with `.rb`) passed in the command-line.
+    module CodeLoader
+      
+      # Loads all code, whether from a deploy pack or additionally
+      # passed on the command line.
+      def load_args
+        (args_to_load || []).each do |path|
+          load_ruby_file(path)
+        end
+      end
+
+      private
+      
+      # Load any additional code that we found out about on the
+      # command-line.
+      #
+      # @return [Array<String>] paths to load culled from the ARGV.
+      def args_to_load
+        ruby_file_args || []
+      end
+
+      # Returns all pre-resolved arguments which are Ruby files.
+      #
+      # @return [Array<String>]
+      def ruby_file_args
+        ARGV.find_all { |arg| arg.to_s =~ /\.rb$/ && arg.to_s !~ /^--/ }
+      end
+      
+      # Loads a single Ruby file, capturing LoadError and SyntaxError
+      # and raising Wukong::Error instead (so it can be easily captured
+      # by the Runner).
+      #
+      # @param [String] path
+      # @raise [Wukong::Error] if there is an error
+      def load_ruby_file path
+        return unless path
+        begin
+          Kernel.load path
+        rescue LoadError, SyntaxError => e
+          raise Error.new(e)
+        end
+      end
+    end
+  end
+end

data/lib/wukong/runner/command_runner.rb

@@ -0,0 +1,44 @@
+module Wukong
+  class Runner
+
+    # Provides methods for executing commandlines.
+    module CommandRunner
+
+      private
+
+      # Execute a command composed of the given parts.
+      #
+      # Will print the command instead if the <tt>--dry_run</tt>
+      # option was given.
+      #
+      # Will *not* raise an error if the command fails.
+      #
+      # @param [Array<String>] argv
+      def execute_command(*argv)
+        command = argv.flatten.reject(&:blank?).join(" \\\n    ")
+        if settings[:dry_run]
+          log.info("Dry run:")
+          puts command
+        else
+          output = `#{command}`
+          puts output unless output.empty?
+        end
+      end
+
+      # Execute a command composed of the given parts.
+      #
+      # Will print the command instead if the <tt>--dry_run</tt>
+      # option was given.
+      #
+      # *Will* raise an error if the command fails.
+      #
+      # @param [Array<String>] argv
+      def execute_command!(*argv)
+        execute_command(argv)
+        raise Error.new("Command failed!") unless $?.success?
+      end
+      
+    end
+
+  end
+end

data/lib/wukong/runner/deploy_pack_loader.rb

@@ -0,0 +1,75 @@
+module Wukong
+  class Runner
+
+    # Lets Wukong bootstrap by requiring an enclosing deploy pack's
+    # environment file if available.
+    #
+    # We use a simple heuristic (presence of 'Gemfile' and
+    # 'config/environment.rb' in a non-root parent directory) to
+    # determine whether or not we are in a deploy pack.
+    module DeployPackLoader
+
+      # Load the actual deploy pack environment.  Will not swallow any
+      # load errors.
+      def load_deploy_pack
+        load_ruby_file(environment_file) if in_deploy_pack?
+      end
+      
+      # Is execution likely happening within a deploy pack?
+      #
+      # See Wukong::Deploy for more information on deploy packs.
+      #
+      # @return [true, false]
+      def in_deploy_pack?
+        return @in_deploy_pack unless @in_deploy_pack.nil?
+        @in_deploy_pack = (deploy_pack_dir != '/')
+      end
+      
+      # Have we already loaded the environment of a deploy pack?
+      #
+      # See Wukong::Deploy for more information on deploy packs.
+      #
+      # @return [true, false]
+      def loaded_deploy_pack?
+        in_deploy_pack? && defined?(::Wukong::Deploy)
+      end
+
+      # The default environment file that will be require'd when
+      # booting.
+      #
+      # @return [String]
+      def environment_file
+        File.join(deploy_pack_dir, 'config', 'environment.rb')
+      end
+      
+      # Return the directory of the enclosing deploy pack.  Will return
+      # the root ('/') if no deeper directory is identified as a deploy
+      # pack.
+      #
+      # @return [String]
+      def deploy_pack_dir
+        return File.dirname(ENV["BUNDLE_GEMFILE"]) if ENV["BUNDLE_GEMFILE"] && is_deploy_pack_dir?(File.dirname(ENV["BUNDLE_GEMFILE"]))
+        return @deploy_pack_dir if @deploy_pack_dir
+        wd     = Dir.pwd
+        parent = File.dirname(wd)
+        until wd == parent
+          return wd if is_deploy_pack_dir?(wd)
+          wd     = parent
+          parent = File.dirname(wd)
+        end
+        @deploy_pack_dir = wd
+      end
+
+      private
+
+      # Could `dir` be a deploy pack dir?
+      #
+      # @param [String] dir
+      # @return [true, false]
+      def is_deploy_pack_dir? dir
+        dir && !dir.empty? && File.directory?(dir) && File.exist?(File.join(dir, 'Gemfile')) && File.exist?(File.join(dir, 'config', 'environment.rb'))
+      end
+    end
+    
+  end
+end

data/lib/wukong/runner/help_message.rb

@@ -0,0 +1,42 @@
+module Wukong
+  class Runner
+
+    # Defines methods for handling help messages.
+    #
+    # Runners which want to modify how help messages are generated
+    # should override the `contextualize_help_message!` instance method.
+    module HelpMessage
+
+      # Was the `--help` option specified on the command line?
+      #
+      # The boot sequence for a Runner strips out the `--help` option to
+      # allow individual Runner classes to customize their help
+      # messages.
+      #
+      # @return [true, false]
+      def help_given?
+        !!@help_given
+      end
+
+      # Strip the `--help` message from the original ARGV, storing
+      # whether or not it was given for later.
+      def strip_help_param!
+        @help_given = ARGV.delete('--help')
+      end
+
+      # Print a help message.
+      def dump_help
+        settings.dump_help
+      end
+
+      # Print a help message and exit.
+      #
+      # @raise [SystemExit]
+      def dump_help_and_exit!
+        dump_help
+        exit(1)
+      end
+        
+    end
+  end
+end

data/lib/wukong/source.rb

@@ -0,0 +1,33 @@
+module Wukong
+
+  # Provides a runner for periodically triggering a dataflow or
+  # processor.
+  module Source
+    include Plugin
+
+    # Configures the given +settings+ object with all settings
+    # specific to Wukong::Source for the given program +name+.
+    #
+    # @param [Configliere::Param] settings the settings to configure
+    # @param [String] program the name of the currently executing program
+    def self.configure settings, program
+      case program
+      when 'wu-source'
+        settings.define :per_sec,    description: "Number of events produced per second", type: Float
+        settings.define :period,     description: "Number of seconds between events (overrides --per_sec)", type: Float
+        settings.define :batch_size, description: "Trigger a finalize across the dataflow each time this many records are processed", type: Integer
+      end
+    end
+
+    # Boots Wukong::Source using the given +settings+ at the given
+    # +root.
+    #
+    # @param [Configliere::Param] settings the settings to use to boot
+    # @param [String] root the root directory to boot in
+    def self.boot(settings, root)
+    end
+    
+  end
+end
+
+require_relative('source/source_runner')

data/lib/wukong/source/source_driver.rb

@@ -0,0 +1,74 @@
+module Wukong
+  module Source
+
+    # A driver which works just like the `Wukong::Local::StdioDriver`
+    # except it ignores input from `STDIN` and instead generates its
+    # own input records according to some periodic schedule.  Each
+    # consecutive record produced will be an incrementing positive
+    # integer (as a string), starting with '1'.
+    class SourceDriver < Wukong::Local::StdioDriver
+      
+      include Logging
+
+      # The index of the record.
+      attr_accessor :index
+
+      # The number of records after which a `Processor#finalize` will
+      # be called.
+      attr_accessor :batch_size
+
+      # Sets the initial value of `index` to 1 and sets the batch size
+      # (only if it's positive).
+      def post_init
+        super()
+        self.index = 1
+        self.batch_size = settings[:batch_size].to_i if settings[:batch_size] && settings[:batch_size].to_i > 0
+      end
+
+      # Starts periodically feeding the processor or dataflow given by
+      # `label` using the given `settings`.
+      #
+      # @param [String, Symbol] label
+      # @param [Configliere::Param, Hash] settings
+      def self.start(label, settings={})
+        driver = new(:foobar, label, settings) # i don't think the 1st argument matters here...
+        driver.post_init
+
+        period = case
+        when settings[:period]  then settings[:period]
+        when settings[:per_sec] then (1.0 / settings[:per_sec]) rescue 1.0
+        else 1.0
+        end
+        driver.create_event
+        EventMachine::PeriodicTimer.new(period) { driver.create_event }
+      end
+
+      # Creates a new event using the following steps:
+      #
+      # 1. Feeds a record with the existing `index` to the dataflow.
+      # 2. Increments the `index`.
+      # 3. Finalizes the dataflow if the number of records is a
+      #    multiple of the `batch_size`.
+      #
+      # @see DriverMethods
+      def create_event
+        receive_line(index.to_s)
+        self.index += 1
+        finalize_dataflow if self.batch_size && (self.index % self.batch_size) == 0
+      end
+
+      # Outputs a `record` from the dataflow or processor to `STDOUT`.
+      #
+      # `STDOUT` will automatically be flushed to force output to
+      # prevent the feeling of "no output" when the looping period is
+      # long.
+      #
+      # @param [Object] record the record yielded by the processor or the terminal node(s) of the dataflow
+      def process record
+        $stdout.puts record
+        $stdout.flush
+      end
+      
+    end
+  end
+end