emque-consuming 1.0.0.beta4

data/lib/emque/consuming/adapters/rabbit_mq.rb

@@ -0,0 +1,26 @@
+module Emque
+  module Consuming
+    module Adapters
+      module RabbitMq
+        def self.default_options
+          {
+            :url => "amqp://guest:guest@localhost:5672",
+            :prefetch => nil,
+            :durable => true,
+            :auto_delete => false
+          }
+        end
+
+        def self.load
+          require_relative "rabbit_mq/manager"
+          require_relative "rabbit_mq/worker"
+          require_relative "rabbit_mq/retry_worker"
+        end
+
+        def self.manager
+          Emque::Consuming::Adapters::RabbitMq::Manager
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/application.rb

@@ -0,0 +1,112 @@
+require "emque/consuming/core"
+require "emque/consuming/actor"
+require "emque/consuming/consumer"
+require "emque/consuming/error_tracker"
+require "emque/consuming/message"
+
+def emque_autoload(klass, file)
+  Kernel.autoload(klass, file)
+end
+
+module Emque
+  module Consuming
+    class ConfigurationError < StandardError; end
+
+    module Application
+      def self.included(descendant)
+        Emque::Consuming.application = descendant
+
+        descendant.class_eval do
+          extend Emque::Consuming::Core
+          include Emque::Consuming::Helpers
+
+          attr_reader :error_tracker, :manager
+
+          private :ensure_adapter_is_configured!, :initialize_error_tracker,
+                  :initialize_manager, :log_prefix, :handle_shutdown
+        end
+      end
+
+      def initialize
+        self.class.instance = self
+
+        logger.info "#{log_prefix}: initializing"
+
+        ensure_adapter_is_configured!
+
+        initialize_manager
+        initialize_error_tracker
+      end
+
+      def notice_error(context)
+        error_tracker.notice_error_for(context)
+        verify_error_status
+      end
+
+      def restart
+        stop
+        initialize_manager
+        error_tracker.occurrences.clear
+        start
+      end
+
+      def start
+        logger.info "#{log_prefix}: starting"
+        manager.async.start
+      end
+
+      def stop
+        logger.info "#{log_prefix}: stopping"
+        manager.stop
+      end
+
+      def verify_error_status
+        if error_tracker.limit_reached?
+          handle_shutdown
+          runner.stop
+        end
+      end
+
+      # private
+
+      def ensure_adapter_is_configured!
+        if config.adapter.nil?
+          raise AdapterConfigurationError,
+                "Adapter not found! use config.set_adapter(name, options)"
+        end
+      end
+
+      def handle_shutdown
+        context = {
+          :limit => error_tracker.limit,
+          :expiration => error_tracker.expiration,
+          :occurrences => error_tracker.occurrences,
+          :status => runner.status.to_h,
+          :configuration => config.to_h
+        }
+
+        Emque::Consuming.logger.error("Error limit exceeded... shutting down")
+        Emque::Consuming.logger.error(context)
+
+        Emque::Consuming.config.shutdown_handlers.each do |handler|
+          handler.call(context)
+        end
+      end
+
+      def initialize_error_tracker
+        @error_tracker = Emque::Consuming::ErrorTracker.new(
+          :expiration => config.error_expiration,
+          :limit => config.error_limit
+        )
+      end
+
+      def initialize_manager
+        @manager = config.adapter.manager.new
+      end
+
+      def log_prefix
+        "#{config.app_name.capitalize} Application"
+      end
+    end
+  end
+end

data/lib/emque/consuming/cli.rb

@@ -0,0 +1,140 @@
+require "optparse"
+require "emque/consuming"
+require "emque/consuming/generators/application"
+
+module Emque
+  module Consuming
+    class Cli
+      APP_CONFIG_FILE = "config/application.rb"
+      COMMANDS = [:console, :new, :start, :stop]
+      IP_REGEX = /^(\d{1,3}\.){3}\d{1,3}$/
+
+      attr_reader :options
+
+      def initialize(argv)
+        self.argv = argv
+
+        extract_command
+        intercept_help
+
+        load_app
+        setup_options
+        parse_options
+
+        execute
+      end
+
+      private
+
+      attr_reader :parser, :command
+      attr_accessor :argv, :runner
+
+      def execute
+        if command == :new
+          Emque::Consuming::Generators::Application.new(options, argv.last).generate
+        else
+          self.runner = Emque::Consuming::Runner.new(options)
+          runner.send(command)
+        end
+      end
+
+      def extract_command
+        if argv.size > 1 and argv[-2] == "new"
+          @command = :new
+        elsif argv.size > 0
+          @command = argv[-1].to_sym
+        end
+      end
+
+      def intercept_help
+        if command == :new and argv.last.to_sym == command
+          argv << "--help"
+        elsif ! COMMANDS.include?(command)
+          argv << "--help"
+        end
+      end
+
+      def load_app
+        current_dir = Dir.pwd
+
+        if File.exist?(File.join(current_dir, APP_CONFIG_FILE))
+          require_relative File.join(current_dir, APP_CONFIG_FILE)
+        end
+      end
+
+      def parse_options
+        parser.parse!(argv)
+      end
+
+      def setup_options
+        @options = {
+          :daemon => false
+        }
+
+        @parser = OptionParser.new { |o|
+          o.on("-P", "--pidfile PATH", "Store pid in PATH") do |arg|
+            options[:pidfile] = arg
+          end
+
+          o.on(
+            "-S",
+            "--socket PATH",
+            "PATH to the application's unix socket"
+          ) do |arg|
+            options[:socket_path] = arg
+          end
+
+          o.on(
+            "-b",
+            "--bind IP:PORT",
+            "IP & port for the http status application to listen on."
+          ) do |arg|
+            ip, port = arg.split(":")
+            port = port.to_i
+            options[:status_host] = ip if ip =~ IP_REGEX
+            options[:status_port] = port if port > 0 && port <= 65535
+          end
+
+          o.on("-d", "--daemon", "Daemonize the application") do
+            options[:daemon] = true
+          end
+
+          o.on(
+            "-e",
+            "--error-limit N",
+            "Set the max errors before application suicide"
+          ) do |arg|
+            limit = arg.to_i
+            options[:error_limit] = limit if limit > 0
+          end
+
+          o.on("-s", "--status", "Run the http status application") do
+            options[:status] = :on
+          end
+
+          o.on(
+            "-x",
+            "--error-expiration SECONDS",
+            "Expire errors after SECONDS"
+          ) do |arg|
+            exp = arg.to_i
+            options[:error_expiration] = exp if exp > 0
+          end
+
+          o.on("--app-name NAME", "Run the application as NAME") do |arg|
+            options[:app_name] = arg
+          end
+
+          o.on(
+            "--env (ex. production)",
+            "Set the application environment, overrides EMQUE_ENV"
+          ) do |arg|
+            options[:env] = arg
+          end
+
+          o.banner = "emque <options> (start|stop|new|console|help) <name (new only)>"
+        }
+      end
+    end
+  end
+end

data/lib/emque/consuming/command_receivers/base.rb

@@ -0,0 +1,37 @@
+module Emque
+  module Consuming
+    module CommandReceivers
+      NotImplemented = Class.new(StandardError)
+
+      class Base
+        include Emque::Consuming::Helpers
+
+        def restart
+          stop if running?
+          start
+        end
+
+        def start
+          raise NotImplemented
+        end
+
+        def stop
+          thread.exit if running?
+          status
+        end
+
+        def status
+          thread ? (thread.status || "stopped") : "stopped"
+        end
+
+        private
+
+        attr_reader :thread
+
+        def running?
+          thread && !thread.stop?
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/command_receivers/http_server.rb

@@ -0,0 +1,103 @@
+require "puma/cli"
+require "emque/consuming/command_receivers/base"
+
+module Emque
+  module Consuming
+    module CommandReceivers
+      class HttpServer < Base
+        attr_accessor :puma
+
+        def initialize
+          ENV["RACK_ENV"] = Emque::Consuming.application.emque_env
+          initialize_puma
+        end
+
+        def start
+          puma.options[:app] = Handler.new
+          @thread = Thread.new { puma.run }
+          status
+        end
+
+        private
+
+        def initialize_puma
+          self.puma =
+            Puma::CLI.new(
+              [],
+              Puma::Events.new(
+                Logger.new(:info),
+                Logger.new(:error)
+              )
+            )
+
+          puma.options[:binds] = [
+            "tcp://#{config.status_host}"+
+            ":#{config.status_port}"
+          ]
+
+          puma.define_singleton_method :set_process_title do
+            # we don't want puma to take over the process name
+          end
+        end
+
+        class Handler
+          include Emque::Consuming::Helpers
+
+          def call(env)
+            req = env["REQUEST_URI"].split("/")
+
+            case req[1]
+            when "status"
+              return render_status
+            when "control"
+              case req[2]
+              when "errors"
+                if req[3..-1] && runner.control.errors(*req[3..-1]) == true
+                  return render_status
+                end
+              else
+                if req[2].is_a?(String) &&
+                   app.manager.workers.has_key?(req[2].to_sym) &&
+                   runner.control.workers(*req[2..-1]) == true
+                  return render_status
+                end
+              end
+            end
+
+            render_404
+          end
+
+          def render_404
+            [404, {}, ["Not Found"]]
+          end
+
+          def render_status(additional = {})
+            [
+              200,
+              {},
+              [
+                Oj.dump(
+                  runner.status.to_hsh.merge(additional),
+                  :mode => :compat
+                )
+              ]
+            ]
+          end
+        end
+
+        class Logger
+          attr_accessor :sync, :method
+
+          def initialize(method)
+            self.method = method
+          end
+
+          def puts(str)
+            Emque::Consuming.logger.send(method, str)
+          end
+          alias :write :puts
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/command_receivers/unix_socket.rb

@@ -0,0 +1,169 @@
+require "socket"
+require "emque/consuming/command_receivers/base"
+
+module Emque
+  module Consuming
+    module CommandReceivers
+      class UnixSocket < Base
+
+        def start
+          @thread = new_socket_server
+          status
+        end
+
+        private
+
+        def app_name
+          config.app_name.capitalize
+        end
+
+        def new_socket_server
+          Thread.new {
+            loop do
+              Socket.unix_server_loop(config.socket_path) do |sock, client_addr|
+                begin
+                  receive_command(sock)
+                rescue
+                  # nothing to do but restart the socket
+                ensure
+                  sock.close
+                end
+              end
+            end
+          }
+        end
+
+        def receive_command(sock)
+          req = Oj.load(sock.recv(100000), :symbol_keys => true)
+          handler = Handler.new(req)
+          sock.send(handler.respond, 0)
+        rescue Oj::ParseError => e
+          sock.send(bad_request(handler), 0)
+          log_error(e)
+        rescue NoMethodError => e
+          sock.send(bad_request(handler), 0)
+          log_error(e)
+        rescue ArgumentError => e
+          sock.send(bad_request(handler), 0)
+          log_error(e)
+        rescue => e
+          sock.send(e.inspect, 0)
+          log_error(e)
+        end
+
+        private
+
+        def bad_request(handler)
+          <<-OUT
+The request was not formatted properly.
+We suggest using Emque::Consuming::Transmitter to send a requests.",
+-------
+#{handler.help rescue "Help broken"}
+          OUT
+        end
+
+        def log_error(e)
+          logger.error(e.inspect)
+          e.backtrace.each do |bt|
+            logger.error(bt)
+          end
+        end
+
+        class Handler
+          include Emque::Consuming::Helpers
+
+          COMMANDS = [:configuration, :errors, :restart, :status, :stop]
+
+          def initialize(args:, command:)
+            self.args = args
+            self.command = command.to_sym
+          end
+
+          def help
+            <<-OUT
+#{app_name} Help
+
+# Information
+
+configuration                 # current configuration of the application
+help                          # this menu
+status                        # current status of the application
+
+# Control
+
+errors clear                  # reset the error count to 0
+errors down                   # decrease the acceptable error threshold by 1
+errors expire_after <seconds> # changes the expiration time for future errors
+errors up                     # increase the acceptable error threshold by 1
+errors retry                  # Reprocesses all messages in the error queue
+restart                       # restart all workers
+stop                          # turn the application off
+-------
+            OUT
+          end
+
+          def respond
+            if valid_request?
+              method(command).call(*args)
+            else
+              help
+            end
+          end
+
+          private
+
+          attr_accessor :args, :command
+
+          def app_name
+            config.app_name.capitalize
+          end
+
+          def configuration
+            <<-OUT
+#{app_name} Config
+-------
+#{config.to_hsh.map { |label, value|
+  "#{label}: #{value.inspect}"
+}.join("\n")}
+-------
+            OUT
+          end
+
+          def errors(*args)
+            runner.control.errors(*args) == true ? status : help
+          end
+
+          def restart
+            runner.restart_application
+            "The application was successfully restarted"
+          end
+
+          def status
+            data = runner.status.to_hsh
+            <<-OUT
+#{app_name} Status
+-------
+errors:
+#{data[:errors].map { |attr, val|
+  "  #{attr}: #{val}"
+}.join("\n")}
+workers:
+#{data[:workers].map { |topic, settings|
+  "  #{topic}: #{settings[:count]}"
+}.join("\n")}
+-------
+            OUT
+          end
+
+          def stop
+            runner.stop
+          end
+
+          def valid_request?
+            COMMANDS.include?(command)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/configuration.rb

@@ -0,0 +1,63 @@
+require "logger"
+
+module Emque
+  module Consuming
+    class Configuration
+      attr_accessor :app_name, :adapter, :error_handlers, :error_limit,
+        :error_expiration, :status, :status_port, :status_host, :socket_path,
+        :shutdown_handlers
+      attr_writer :env, :log_level
+
+      def initialize
+        @app_name = ""
+        @error_handlers     = []
+        @error_limit        = 5
+        @error_expiration   = 3600 # 60 minutes
+        @log_level          = nil
+        @status_port        = 10000
+        @status_host        = "0.0.0.0"
+        @status             = :off # :on
+        @socket_path        = "tmp/emque.sock"
+        @shutdown_handlers  = []
+      end
+
+      def env
+        Emque::Consuming.application.emque_env
+      end
+
+      def env_var
+        @env
+      end
+
+      def log_level
+        @log_level ||= Logger::INFO
+      end
+
+      def set_adapter(name, options = {})
+        @adapter = Emque::Consuming::Adapter.new(name, options)
+      end
+
+      def to_hsh
+        {}.tap { |config|
+          [
+            :app_name,
+            :adapter,
+            :env,
+            :error_handlers,
+            :error_limit,
+            :error_expiration,
+            :log_level,
+            :status_port,
+            :status_host,
+            :status,
+            :socket_path,
+            :shutdown_handlers
+          ].each { |attr|
+            config[attr] = send(attr)
+          }
+        }
+      end
+      alias :to_h :to_hsh
+    end
+  end
+end

data/lib/emque/consuming/consumer/common.rb

@@ -0,0 +1,61 @@
+require "pipe"
+
+module Emque
+  module Consuming
+    def self.consumer
+      Module.new do
+        define_singleton_method(:included) do |descendant|
+          descendant.send(:include, ::Pipe)
+          descendant.send(:include, ::Emque::Consuming::Consumer::Common)
+        end
+      end
+    end
+
+    class Consumer
+      module Common
+        def self.included(descendant)
+          descendant.class_eval do
+            attr_reader :message
+            private :handle_error, :pipe
+          end
+        end
+
+        def consume(handler_method, message)
+          send(handler_method, message)
+        end
+
+        def pipe_config
+          @pipe_config ||= Pipe::Config.new(
+            :error_handlers => [method(:handle_error)],
+            :stop_on => ->(msg, _, _) { !(msg.respond_to?(:continue?) && msg.continue?) }
+          )
+        end
+
+        def handle_error(e, method:, subject:)
+          context = {
+            :consumer => self.class.name,
+            :message => {
+              :current => subject.values,
+              :original => subject.original
+            },
+            :offset => subject.offset,
+            :partition => subject.partition,
+            :pipe_method => method,
+            :topic => subject.topic
+          }
+
+          # log the error by default
+          Emque::Consuming.logger.error("Error consuming message #{e}")
+          Emque::Consuming.logger.error(context)
+          Emque::Consuming.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
+
+          Emque::Consuming.config.error_handlers.each do |handler|
+            handler.call(e, context)
+          end
+
+          Emque::Consuming.application.instance.notice_error(context)
+        end
+      end
+    end
+  end
+end