oni 0.0.1 → 1.0.0

data/lib/oni.rb

@@ -1,2 +1,8 @@
+require 'thread'
+require 'benchmark'
+
 require_relative 'oni/version'
-require_relative 'oni/thread_pool'
+require_relative 'oni/configurable'
+require_relative 'oni/mapper'
+require_relative 'oni/worker'
+require_relative 'oni/daemon'

data/lib/oni/configurable.rb

@@ -0,0 +1,107 @@
+module Oni
+  ##
+  # Configurable is a basic configuration mixin that can be used to set options
+  # on class level and easily access them on instance level, optionally only
+  # evaluating the setting when it's accessed.
+  #
+  # Basic usage:
+  #
+  #     class SomeClass
+  #       include Oni::Configurable
+  #
+  #       set :threads, 5
+  #       set :logger, proc { Logger.new(STDOUT) }
+  #
+  #       def some_method
+  #         option(:threads).times do
+  #           # ...
+  #         end
+  #       end
+  #     end
+  #
+  module Configurable
+    ##
+    # @param [Class|Module] into
+    #
+    def self.included(into)
+      into.extend(ClassMethods)
+    end
+
+    ##
+    # Returns the value of the given option. If the value responds to `#call`
+    # the method is invoked and the return value of this call is returned.
+    #
+    # @param [Symbol|String] name
+    # @param [Mixed] default The default value to return if no custom one was
+    #  found.
+    # @return [Mixed]
+    #
+    def option(name, default = nil)
+      value = self.class.options[name.to_sym]
+
+      if default and !value
+        value = default
+      end
+
+      return value.respond_to?(:call) ? value.call : value
+    end
+
+    ##
+    # Raises an error if the given option isn't set.
+    #
+    # @param [Symbol|String] option
+    # @raise [ArgumentError]
+    #
+    def require_option!(option)
+      unless option(option)
+        raise ArgumentError, "The option #{option} is required but isn't set"
+      end
+    end
+
+    module ClassMethods
+      ##
+      # Returns a Hash containing the options of the current class.
+      #
+      # @return [Hash]
+      #
+      def options
+        return @options ||= {}
+      end
+
+      ##
+      # Sets the option to the given value. If a Proc (or any object that
+      # responds to `#call`) is given it's not evaluated until it's accessed.
+      # This makes it possible to for example set a logger that's not created
+      # until an instance of the including class is created.
+      #
+      # @example Setting a regular option
+      #  set :number, 10
+      #
+      # @example Setting an option using a proc
+      #  # This means the logger won't be shared between different instances of
+      #  # the including class.
+      #  set :logger, proc { Logger.new(STDOUT) }
+      #
+      # @param [Symbol|String] option
+      # @param [Mixed] value
+      #
+      def set(option, value)
+        options[option.to_sym] = value
+      end
+
+      ##
+      # Sets a number of options based on the given Hash.
+      #
+      # @example
+      #  set_multiple(:a => 10, :b => 20)
+      #
+      # @param [Hash] options
+      #
+      def set_multiple(options)
+        options.each do |option, value|
+          set(option, value)
+        end
+      end
+    end # ClassMethods
+  end # Configurable
+end # Oni

data/lib/oni/daemon.rb

@@ -0,0 +1,190 @@
+module Oni
+  ##
+  # The Daemon class takes care of retrieving work to be processed, scheduling
+  # it and dispatching it to a mapper and worker. In essence a Daemon instance
+  # can be seen as a controller when compared with typical MVC frameworks.
+  #
+  # This daemon starts a number of threads (5 by default) that will each
+  # perform work on their own using the corresponding mapper and worker class.
+  #
+  # @!attribute [r] workers
+  #  @return [Array<Thread>]
+  #
+  class Daemon
+    include Configurable
+
+    attr_reader :workers
+
+    ##
+    # The default amount of threads to start.
+    #
+    # @return [Fixnum]
+    #
+    DEFAULT_THREAD_AMOUNT = 5
+
+    ##
+    # Creates a new instance of the class and calls `#after_initialize` if it
+    # is defined.
+    #
+    def initialize
+      @workers = []
+
+      after_initialize if respond_to?(:after_initialize)
+    end
+
+    ##
+    # Starts the daemon and waits for all threads to finish execution. This
+    # method is blocking since it will wait for all threads to finish.
+    #
+    # If the current class has a `before_start` method defined it's called
+    # before starting the daemon.
+    #
+    def start
+      before_start if respond_to?(:before_start)
+
+      # If we don't have any threads run in non threaded mode.
+      if threads > 0
+        threads.times do
+          workers << spawn_thread
+        end
+
+        workers.each(&:join)
+      else
+        run_thread
+      end
+    rescue => error
+      error(error)
+    end
+
+    ##
+    # Terminates all the threads and clears up the list. Note that calling this
+    # method acts much like sending a SIGKILL signal to a process: threads will
+    # be shut down *immediately*.
+    #
+    def stop
+      workers.each(&:kill)
+      workers.clear
+    end
+
+    ##
+    # Returns the amount of threads to use.
+    #
+    # @return [Fixnum]
+    #
+    def threads
+      return option(:threads, DEFAULT_THREAD_AMOUNT)
+    end
+
+    ##
+    # Processes the given message. Upon completion the `#complete` method is
+    # called and passed the resulting output.
+    #
+    # @param [Mixed] message
+    #
+    def process(message)
+      output  = nil
+      timings = Benchmark.measure do
+        output = run_worker(message)
+      end
+
+      complete(message, output, timings)
+    end
+
+    ##
+    # Maps the input, runs the worker and then maps the output into something
+    # that the daemon can understand.
+    #
+    # @param [Mixed] message
+    # @return [Mixed]
+    #
+    def run_worker(message)
+      mapper = create_mapper
+      input  = mapper.map_input(message)
+      worker = option(:worker).new(*input)
+      output = worker.process
+
+      return mapper.map_output(output)
+    end
+
+    ##
+    # Receives a message, by default this method raises an error.
+    #
+    # @raise [NotImplementedError]
+    #
+    def receive
+      raise NotImplementedError, 'You must manually implement #receive'
+    end
+
+    ##
+    # Called when a job has been completed, by default this method is a noop.
+    # This method is passed 3 arguments:
+    #
+    # 1. The raw input message.
+    # 2. The output of the worker (remapped by the mapper).
+    # 3. A Benchmark::Tms instance that contains the timings for processing the
+    #    message.
+    #
+    # @param [Mixed] message The raw input message (e.g. an AWS SQS message)
+    # @param [Mixed] output The output of the worker.
+    # @param [Benchmark::Tms] timings
+    #
+    def complete(message, output, timings)
+    end
+
+    ##
+    # Called whenever an error is raised in the daemon, mapper or worker. By
+    # default this method just re-raises the error.
+    #
+    # Note that this callback method is called from a thread in which the
+    # exception occured, not from the main thread.
+    #
+    # @param [StandardError] error
+    #
+    def error(error)
+      raise error
+    end
+
+    ##
+    # Creates a new mapper and passes it a set of arguments as defined in
+    # {Oni::Daemon#mapper_arguments}.
+    #
+    # @return [Oni::Mapper]
+    #
+    def create_mapper
+      unless option(:mapper)
+        raise ArgumentError, 'No mapper has been set in the `:mapper` option'
+      end
+
+      return option(:mapper).new
+    end
+
+    ##
+    # Spawns a new thread that waits for daemon input.
+    #
+    # @return [Thread]
+    #
+    def spawn_thread
+      thread = Thread.new { run_thread }
+
+      thread.abort_on_exception = true
+
+      return thread
+    end
+
+    ##
+    # The main code to execute in individual threads.
+    #
+    # If an error occurs in the receive method or processing a job the error
+    # handler is executed and the process is retried. It's the responsibility
+    # of the `error` method to determine if the process should fail only once
+    # (and fail hard) or if it should continue running.
+    #
+    def run_thread
+      receive { |message| process(message) }
+    rescue => error
+      error(error)
+
+      retry
+    end
+  end # Daemon
+end # Oni

data/lib/oni/daemons/sqs.rb

@@ -0,0 +1,61 @@
+require 'aws-sdk'
+
+module Oni
+  module Daemons
+    ##
+    # The SQS daemon is a basic daemon skeleton that can be used to process
+    # jobs from an Amazon SQS queue.
+    #
+    # Basic usage:
+    #
+    #     class MyDaemon < Oni::Daemons::SQS
+    #       set :queue_name, 'my_queue'
+    #     end
+    #
+    # The following options can be set:
+    #
+    # * `queue_name` (required): the name of the queue to poll as a String.
+    # * `poll_options`: a Hash of options to pass to the `poll` method of the
+    #   AWS SQS queue. See the documentation of `AWS::SQS::Queue#poll` for more
+    #   information on the available options.
+    #
+    class SQS < Daemon
+      ##
+      # Checks if the `queue_name` option is set.
+      #
+      def after_initialize
+        require_option!(:queue_name)
+      end
+
+      ##
+      # Polls an SQS queue for a message and processes it.
+      #
+      def receive
+        queue.poll(poll_options) do |message|
+          yield message
+        end
+      end
+
+      ##
+      # Returns a Hash containing the options to use for the `poll` method of
+      # the SQS queue.
+      #
+      # @return [Hash]
+      #
+      def poll_options
+        return option(:poll_options, {})
+      end
+
+      ##
+      # Returns the queue to use for the current thread.
+      #
+      # @return [AWS::SQS::Queue]
+      #
+      #:nocov:
+      def queue
+        return AWS::SQS.new.queues.named(option(:queue_name))
+      end
+      #:nocov:
+    end # SQS
+  end # Daemons
+end # Oni

data/lib/oni/mapper.rb

@@ -0,0 +1,30 @@
+module Oni
+  ##
+  # Abstract mapper class that takes care of some common boilerplate code.
+  #
+  class Mapper
+    include Configurable
+
+    ##
+    # Remaps the input of the daemon into a format that's easy to use for the
+    # worker.
+    #
+    # @param [Mixed] input
+    # @return [Mixed]
+    #
+    def map_input(input)
+      return input
+    end
+
+    ##
+    # Remaps the output of the working into a format that's easy to use for the
+    # daemon layer.
+    #
+    # @param [Mixed] output
+    # @return [Mixed]
+    #
+    def map_output(output)
+      return output
+    end
+  end # Mapper
+end # Oni

data/lib/oni/version.rb

@@ -1,3 +1,3 @@
 module Oni
-  VERSION = '0.0.1'
+  VERSION = '1.0.0'
 end # Oni

data/lib/oni/worker.rb

@@ -0,0 +1,18 @@
+module Oni
+  ##
+  # An abstract worker class that takes care of some common boilerplate code.
+  #
+  class Worker
+    include Configurable
+
+    ##
+    # Runs the worker and returns some kind of output.
+    #
+    # @return [Mixed]
+    # @raise [NotImplementedError]
+    #
+    def process
+      raise NotImplementedError, 'You must implement #process yourself'
+    end
+  end # Worker
+end # Oni

data/oni.gemspec

@@ -0,0 +1,30 @@
+require File.expand_path('../lib/oni/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name    = 'oni'
+  gem.version = Oni::VERSION
+  gem.authors = [
+    'Yorick Peterse',
+    'Wilco van Duinkerken'
+  ]
+
+  gem.summary     = 'Framework for building concurrent daemons in Ruby.'
+  gem.description = gem.summary
+  gem.has_rdoc    = 'yard'
+  gem.license     = 'MIT'
+
+  gem.required_ruby_version = '>= 1.9.3'
+
+  gem.files       = `git ls-files`.split("\n").sort
+  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files  = gem.files.grep(%r{^(test|spec|features)/})
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'bundler'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'yard'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'kramdown'
+  gem.add_development_dependency 'ci_reporter'
+  gem.add_development_dependency 'aws-sdk'
+end