opener-daemons 1.3.0 → 2.0

data/lib/opener/daemons/daemon.rb

@@ -1,229 +1,120 @@
-# encoding: UTF-8
+module Opener
+  module Daemons
+    ##
+    # The Daemon class communicates with an AWS SQS queue and delegates work to
+    # the mapper and worker classes.
+    #
+    # @!attribute [r] component
+    #  @return [Class]
+    #
+    class Daemon < Oni::Daemons::SQS
+      attr_reader :component
 
-require 'thread'
-require 'opener/daemons/sqs'
-require 'json'
+      set :worker, Worker
+      set :mapper, Mapper
 
-Encoding.default_internal = Encoding::UTF_8
-Encoding.default_external = Encoding::UTF_8
+      # The name of the SQS input queue to use.
+      set :queue_name, proc { Daemons.input_queue }
 
-module Opener
-  module Daemons
-    class Daemon
-      attr_reader :batch_size, :buffer_size, :sleep_interval,
-                  :input_queue, :output_queue,
-                  :input_buffer, :output_buffer,
-                  :bucket_name, :bucket_dir, :file_suffix,
-                  :klass,
-                  :logger,
-                  :script_name
-
-      attr_accessor :threads, :thread_counts
-
-      def initialize(klass, options={})
-
-        @threads = {:readers=>[], :workers=>[], :writers=>[], :reporters=>[]}
-        @thread_counts = {:readers => options.fetch(:readers, 1),
-                          :workers => options.fetch(:workers, 5),
-                          :writers => options.fetch(:writers, 1)}
-
-        @relentless = options.fetch(:relentless, false)
-        @sleep_interval = options.fetch(:sleep_interval, 5)
-
-        # Initialize queues
-        @input_queue  = Opener::Daemons::SQS.find(options.fetch(:input_queue))
-        if options[:output_queue]
-          @output_queue = Opener::Daemons::SQS.find(options[:output_queue])
-        end
-        
-        # Get bucket name and other bucket options, if any.
-        if @bucket_name = options[:bucket_name]
-           @bucket_dir  = options.fetch(:bucket_dir, nil)
-           @file_suffix = options.fetch(:file_suffix, nil)
-         end
-
-        # Initialize Buffers
-        @input_buffer  = Queue.new
-        @output_buffer = Queue.new
-
-        # Batch and Buffer size for a smooth flow.
-        @batch_size = options.fetch(:batch_size, 10)
-        @buffer_size = options[:buffer_size]
-
-        # Working component
-        @klass = klass
-
-        @script_name = File.basename($0, ".rb")
-
-        @logger = Logger.new(options.fetch(:log, STDOUT))
-        @logger.level = if options.fetch(:debug, false)
-                          Logger::DEBUG
-                        else
-                          Logger::INFO
-                        end
-
-        logger.debug(options.to_json)
-      end
+      # The amount of threads to use.
+      set :threads, proc { Daemons.daemon_threads }
 
-      def buffer_new_messages
-        return if input_buffer.size > buffer_size
-        return if output_buffer.size > buffer_size
-        messages = input_queue.receive_messages(batch_size)
+      ##
+      # @param [Class] component The component to run in the worker.
+      #
+      def initialize(component)
+        @component = component
 
-        if messages.nil?
-          sleep(sleep_interval)
-          return
-        end
-        messages.each do |message|
-          input_buffer << message
-        end
+        super() # keep parenthesis, parent method doesn't take arguments.
       end
 
-      def start
-        Thread.abort_on_exception = true
+      ##
+      # Called before the daemon is started.
+      #
+      def before_start
+        Syslog.open(ENV['APP_NAME'], ::Syslog::LOG_CONS | ::Syslog::LOG_PID)
 
-        start_readers
-        start_workers
-        start_writers
-        start_reporters
+        Syslog.info(
+          'Starting daemon',
+          :queue   => option(:queue_name),
+          :threads => threads
+        )
 
-        threads[:readers].each(&:join)
-        threads[:workers].each(&:join)
-        threads[:writers].each(&:join)
-        threads[:reporters].each(&:join)
-      end
+        GC::Profiler.enable
 
-      def start_readers
-        thread_counts[:readers].times do |t|
-          threads[:readers] << Thread.new do
-            logger.info "Reader #{t+1} ready for action..."
-            loop do
-              buffer_new_messages
-            end
-          end
-        end
-      end
+        Daemons.configure_rollbar
 
-      def start_workers
-        thread_counts[:workers].times do |t|
-          threads[:workers] << Thread.new do
-            logger.info "Worker #{t+1} launching..."
-            identifier = klass.new
-            loop do
-              message = input_buffer.pop
-              
-              input = get_input(message[:body])
-              input,* = input if input.kind_of?(Array)
-
-              begin
-                output, * = identifier.run(input)
-                if output.empty?
-                  raise "The component returned an empty response."
-                end
-              rescue Exception => e
-                if relentless?
-                  raise
-                else
-                  logger.error(e)
-                  output = input
-                end
-              end
-              message[:body].delete("input")
-              output_buffer.push({ :output=>output, 
-                                   :body => message[:body],
-                                   :handle=>message[:receipt_handle]
-                                   })
-            end
-          end
-        end
+        NewRelic::Agent.manual_start if Daemons.newrelic?
       end
 
-      def start_writers
-        thread_counts[:writers].times do |t|
-          threads[:writers] << Thread.new do
-            logger.info "Pusher #{t+1} ready for action..."
-            loop do
-              message = output_buffer.pop
-              callbacks = extract_callbacks(message[:body]["callbacks[]"])
-              handler = Opener::CallbackHandler.new
-              
-              if bucket_name
-                filename = [message[:body]["request_id"], script_name, Time.now.to_i].join("-")
-                s3 = Opener::Daemons::S3.new(bucket_name, message[:output].force_encoding("UTF-8"), filename,  bucket_dir, file_suffix)
-                s3.upload
-                message[:body][:input_url] = s3.url
-              else
-                message[:body][:input] = message[:output].force_encoding("UTF-8")
-              end
-              
-              
-              unless callbacks.empty?
-                callback_url = callbacks.shift
-                message[:body][:'callbacks[]'] = callbacks
-                payload = {:body => message[:body]}
-                handler.post(callback_url, payload)
-              else
-                payload = {:body => message[:body]}
-                handler.post(output_queue.queue_url, payload)
-              end
-              input_queue.delete_message(message[:handle])
-              
-            end
-          end
+      ##
+      # Overwrites the original method so that we can inject the component into
+      # the mapper.
+      #
+      # @see [Oni::Daemon#create_mapper]
+      #
+      def create_mapper
+        unless option(:mapper)
+          raise ArgumentError, 'No mapper has been set in the `:mapper` option'
         end
-      end
 
-      def start_reporters
-        threads[:reporters] << Thread.new do
-          loop do
-            log = {:buffers=>{:input=>input_buffer.size}}
-            log[:buffers][:output] = output_buffer.size if output_buffer
+        return option(:mapper).new(component)
+      end
 
-            logger.debug log.to_json
-            sleep(2)
-          end
-        end
+      ##
+      # Called when an error occurs.
+      #
+      # @param [StandardError] error
+      #
+      def error(error)
+        error, params = unwrap_error(error)
 
-        threads[:reporters] << Thread.new do
-          loop do
-            thread_types = threads.keys - [:reporters]
-            thread_counts = thread_types.map do |type|
-              threads[type].select{|thread| thread.status}.count
-            end
-            zip = thread_types.zip(thread_counts)
-            logger.debug "active thread counts: #{zip}"
-
-            sleep(10)
-          end
-        end
+        report_exception(error, params)
       end
 
-      def buffer_size
-        @buffer_size ||= (4 * batch_size)
+      ##
+      # @param [AWS::SQS::ReceivedMessage] message
+      # @param [Mixed] output
+      # @param [Benchmark::Tms] timings
+      #
+      def complete(message, output, timings)
+        log_msg = "Finished message #{message.id}"
+
+        Syslog.info(log_msg)
       end
 
-      def relentless?
-        @relentless
+      ##
+      # Sends an error to Rollbar.
+      #
+      # @param [StandardError] error
+      # @param [Hash] parameters
+      #
+      def report_exception(error, parameters = {})
+        if Daemons.rollbar?
+          Rollbar.error(
+            error,
+            :active_threads   => Thread.list.count,
+            :ruby_description => RUBY_DESCRIPTION,
+            :parameters       => parameters
+          )
+        else
+          raise error
+        end
       end
-      
+
       ##
-      # Returns an Array containing the callback URLs, ignoring empty values.
+      # Takes either a regular error or a `Oni::WrappedError` and unwraps it,
+      # returning the original error and the parameters (if any).
       #
-      # @param [Array|String] input
+      # @param [StandardError] error
       # @return [Array]
       #
-      def extract_callbacks(input)
-        return [] if input.nil? || input.empty?
+      def unwrap_error(error)
+        params = error.respond_to?(:parameters) ? error.parameters : {}
+        error  = error.original_error if error.respond_to?(:original_error)
 
-        callbacks = input.compact.reject(&:empty?)
-
-        return callbacks
-      end
-      
-      def get_input(body)
-        return body.delete("input") if body["input"]
-        return HTTPClient.new.get(body.delete("input_url")).body if body["input_url"]
+        return error, params
       end
-    end
-  end
-end
+    end # Daemon
+  end # Daemons
+end # Opener

data/lib/opener/daemons/daemons.rb

@@ -0,0 +1,75 @@
+module Opener
+  module Daemons
+    ##
+    # Returns `true` if New Relic monitoring should be enabled.
+    #
+    # @return [TrueClass|FalseClass]
+    #
+    def self.newrelic?
+      return !!ENV['NEWRELIC_TOKEN']
+    end
+
+    ##
+    # Returns `true` if Rollbar error tracking should be enabled.
+    #
+    # @return [TrueClass|FalseClass]
+    #
+    def self.rollbar?
+      return !!ENV['ROLLBAR_TOKEN']
+    end
+
+    ##
+    # Returns `true` if Syslog should be enabled.
+    #
+    # @return [TrueClass|FalseClass]
+    #
+    def self.syslog?
+      return !!ENV['ENABLE_SYSLOG']
+    end
+
+    ##
+    # Returns the name of the input queue to use.
+    #
+    # @return [String]
+    #
+    def self.input_queue
+      return ENV['INPUT_QUEUE']
+    end
+
+    ##
+    # The name of the S3 bucket to store output in.
+    #
+    # @return [String]
+    #
+    def self.output_bucket
+      return ENV['OUTPUT_BUCKET']
+    end
+
+    ##
+    # Returns the amount of daemon threads to run.
+    #
+    # @return [Fixnum]
+    #
+    def self.daemon_threads
+      return ENV['DAEMON_THREADS'].to_i
+    end
+
+    ##
+    # Configures Rollbar.
+    #
+    def self.configure_rollbar
+      Rollbar.configure do |config|
+        config.access_token = ENV['ROLLBAR_TOKEN']
+        config.enabled      = rollbar?
+        config.environment  = environment
+      end
+    end
+
+    ##
+    # @return [String]
+    #
+    def self.environment
+      return ENV['DAEMON_ENV'] || ENV['RACK_ENV'] || ENV['RAILS_ENV']
+    end
+  end # Daemons
+end # Opener

data/lib/opener/daemons/downloader.rb

@@ -0,0 +1,36 @@
+module Opener
+  module Daemons
+    ##
+    # Downloads and validates text/XML documents used as input.
+    #
+    # @!attribute [r] http
+    #  @return [HTTPClient]
+    #
+    class Downloader
+      attr_reader :http
+
+      def initialize
+        @http = HTTPClient.new
+      end
+
+      ##
+      # Downloads the document located at `url`.
+      #
+      # @param [String] url
+      # @return [String]
+      #
+      def download(url)
+        resp = http.get(url, :follow_redirect => true)
+
+        unless resp.ok?
+          raise(
+            HTTPClient::BadResponseError,
+            "Got HTTP #{resp.status}: #{resp.body}"
+          )
+        end
+
+        return resp.body
+      end
+    end # Downloader
+  end # Daemons
+end # Opener

data/lib/opener/daemons/mapper.rb

@@ -0,0 +1,57 @@
+module Opener
+  module Daemons
+    ##
+    # Maps the input/output between the daemon and the worker in such a format
+    # that both ends can work with it easily.
+    #
+    # @!attribute [r] component
+    #  @return [Class]
+    #
+    class Mapper < Oni::Mapper
+      attr_reader :component
+
+      ##
+      # The directory containing JSON schema files.
+      #
+      # @return [String]
+      #
+      SCHEMA_DIRECTORY = File.expand_path('../../../../schema', __FILE__)
+
+      ##
+      # Path to the schema file.
+      #
+      # @return [String]
+      #
+      INPUT_SCHEMA = File.join(SCHEMA_DIRECTORY, 'sqs_input.json')
+
+      ##
+      # @param [Class] component
+      #
+      def initialize(component)
+        @component = component
+      end
+
+      ##
+      # @param [AWS::SQS::ReceivedMessage] message
+      # @return [Hash]
+      #
+      def map_input(message)
+        decoded = JSON(message.body)
+
+        validate_input!(decoded)
+
+        return Configuration.new(component, decoded)
+      end
+
+      ##
+      # Validates the given input Hash.
+      #
+      # @param [Hash] input
+      # @raise [JSON::Schema::ValidationError]
+      #
+      def validate_input!(input)
+        JSON::Validator.validate!(INPUT_SCHEMA, input)
+      end
+    end # Mapper
+  end # Daemons
+end # Opener

data/lib/opener/daemons/option_parser.rb

@@ -0,0 +1,139 @@
+module Opener
+  module Daemons
+    ##
+    # Slop wrapper for parsing daemon options and passing them to the underlying
+    # script.
+    #
+    # @!attribute [r] name
+    #  The name of the program to display in help messages.
+    #  @return [String]
+    #
+    # @!attribute [r] parser
+    #  @return [Slop]
+    #
+    class OptionParser
+      attr_reader :name, :parser
+
+      ##
+      # @param [String] name
+      #
+      def initialize(name)
+        @name   = name
+        @parser = configure_slop
+      end
+
+      ##
+      # @see [Slop#parse]
+      #
+      def parse(*args)
+        return parser.parse(*args)
+      end
+
+      ##
+      # @return [Slop]
+      #
+      def configure_slop
+        # Slop uses instance_eval, which means `self` will point to the `Slop`
+        # instance in the block below.
+        outer       = self
+        daemon_name = "#{name}-daemon"
+        cli_name    = daemon_name.sub('opener-', '')
+
+        # Using :strict => false ensures that unrecognized options are kept in
+        # ARGV.
+        return Slop.new(:strict => false, :indent => 2) do
+          banner "Usage: #{cli_name} <start|stop|restart> [OPTIONS]"
+
+          separator <<-EOF.chomp
+
+About:
+
+    Runs the OpeNER component as a daemon. By default the daemon runs in the
+    foreground but using the "start" command it can detach itself. Output is
+    logged using Syslog, allowing easy distribution and management of log
+    output.
+
+Commands:
+
+    * start: starts the daemon in the background
+    * stop: stops the daemon
+    * restart: restarts the daemon
+
+    Not providing a specific command will result in the daemon running in the
+    foreground.
+
+Environment Variables:
+
+    These daemons make use of Amazon SQS queues and other Amazon services. In
+    order to use these services you should make sure the following environment
+    variables are set:
+
+    * AWS_ACCESS_KEY_ID
+    * AWS_SECRET_ACCESS_KEY
+    * AWS_REGION
+
+    If you're running this daemon on an EC2 instance then the first two
+    environment variables will be set automatically if the instance has an
+    associated IAM profile. The AWS_REGION variable must _always_ be set.
+
+    Optionally you can also set the following extra variables:
+
+    * NEWRELIC_TOKEN: when set the daemon will send profiling data to New Relic
+      using this token. The application name will be "#{daemon_name}".
+
+    * ROLLBAR_TOKEN: when set the daemon will report errors to Rollbar using
+      this token. You can freely use this in combination with NEWRELIC_TOKEN.
+
+Component Options:
+
+    Certain OpeNER components might define their own commandline options. The
+    most common one is the "--resource-path" option which can be used to load
+    (and optionally download) a set of models and/or lexicons.
+
+    Options that are not explicitly defined below will be passed straight to
+    the component. Refer to the documentation of the individual components to
+    see which options are available.
+          EOF
+
+          separator "\nOptions:\n"
+
+          on :h, :help, 'Shows this help message' do
+            abort to_s
+          end
+
+          on :i=,
+            :input=,
+            "The name of the input queue",
+            :as      => String,
+            :default => outer.name
+
+          on :b=,
+            :bucket=,
+            'The S3 bucket to store output in',
+            :as      => String,
+            :default => outer.name
+
+          on :P=,
+            :pidfile=,
+            "Path to the PID file",
+            :as      => String,
+            :default => "/var/run/opener/#{daemon_name}.pid"
+
+          on :t=,
+            :threads=,
+            'The amount of threads to use',
+            :as      => Integer,
+            :default => 10
+
+          on :w=,
+            :wait=,
+            'The amount of seconds to wait for the daemon to start',
+            :as      => Integer,
+            :default => 3
+
+          on :'disable-syslog', 'Disables Syslog logging (enabled by default)'
+        end
+      end
+    end # OptionParser
+  end # Daemons
+end # Opener