emque-consuming 1.0.0.beta4

data/lib/emque/consuming/consumer.rb

@@ -0,0 +1,33 @@
+require "oj"
+require_relative "consumer/common"
+
+module Emque
+  module Consuming
+    class BlockingFailure < StandardError; end
+
+    class Consumer
+      include Emque::Consuming.consumer
+
+      def process(message)
+        pipe(message, :through => [:parse, :route])
+      end
+
+      private
+
+      def parse(message)
+        message.with(
+          :values =>
+            Oj.load(message.original, :symbol_keys => true)
+        )
+      end
+
+      def route(message)
+        Emque::Consuming.application.router.route(
+          message.values.fetch(:metadata).fetch(:topic),
+          message.values.fetch(:metadata).fetch(:type),
+          message
+        )
+      end
+    end
+  end
+end

data/lib/emque/consuming/consuming.rb

@@ -0,0 +1,27 @@
+require "emque/consuming/runner"
+require "emque/consuming/logging"
+
+module Emque
+  module Consuming
+    class << self
+      attr_accessor :application
+
+      # The Configuration instance used to configure the Emque::Consuming environment
+      def config
+        Emque::Consuming.application.config
+      end
+
+      def logger
+        Emque::Consuming::Logging.logger
+      end
+
+      def logger=(log)
+        Emque::Consuming::Logging.logger = log
+      end
+
+      def runner
+        Emque::Consuming::Runner.instance
+      end
+    end
+  end
+end

data/lib/emque/consuming/control/errors.rb

@@ -0,0 +1,41 @@
+module Emque
+  module Consuming
+    class Control
+      class Errors
+        include Emque::Consuming::Helpers
+
+        COMMANDS = [:clear, :down, :expire_after, :up, :retry]
+
+        def clear
+          app.error_tracker.occurrences.clear
+        end
+
+        def down
+          if app.error_tracker.limit > 1
+            config.error_limit = app.error_tracker.limit -= 1
+            app.verify_error_status
+          end
+        end
+
+        def expire_after(seconds)
+          unless seconds.is_a?(Integer)
+            raise ArgumentError, "first argument must be an integer"
+          end
+          config.error_expiration = app.error_tracker.expiration = seconds
+        end
+
+        def up
+          config.error_limit = app.error_tracker.limit += 1
+        end
+
+        def retry
+          app.manager.retry_errors
+        end
+
+        def respond_to?(method)
+          COMMANDS.include?(method.to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/control/workers.rb

@@ -0,0 +1,23 @@
+module Emque
+  module Consuming
+    class Control
+      class Workers
+        include Emque::Consuming::Helpers
+
+        COMMANDS = [:down, :up]
+
+        def down(topic)
+          app.manager.worker(topic: topic.to_sym, command: :down)
+        end
+
+        def up(topic)
+          app.manager.worker(topic: topic.to_sym, command: :up)
+        end
+
+        def respond_to?(method)
+          COMMANDS.include?(method.to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/control.rb

@@ -0,0 +1,33 @@
+require "emque/consuming/control/errors"
+require "emque/consuming/control/workers"
+
+module Emque
+  module Consuming
+    class Control
+      include Emque::Consuming::Helpers
+
+      def initialize
+        @errors = Emque::Consuming::Control::Errors.new
+        @workers = Emque::Consuming::Control::Workers.new
+      end
+
+      def errors(*args)
+        if args[0] && @errors.respond_to?(args[0])
+          @errors.send(args.shift, *args)
+          true
+        else
+          @errors
+        end
+      end
+
+      def workers(topic = nil, command = nil, *args)
+        if command && topic && @workers.respond_to?(command)
+          @workers.send(command, topic, *args)
+          true
+        else
+          @workers
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/core.rb

@@ -0,0 +1,89 @@
+require "celluloid"
+require "inflecto"
+require "emque/consuming/configuration"
+require "emque/consuming/adapter"
+require "emque/consuming/logging"
+require "emque/consuming/router"
+require "emque/consuming/helpers"
+
+module Emque
+  module Consuming
+    module Core
+      def self.extended(descendant)
+        descendant.class_eval do
+          class << self
+            attr_accessor :root, :topic_mapping, :router, :instance
+
+            alias :configure :instance_exec
+          end
+        end
+      end
+
+      def initialize_core!
+        unless root
+          call_stack = caller_locations.map(&:path)
+          self.root = File.expand_path(
+            "../..",
+            call_stack.find { |call| call !~ %r{lib/emque} }
+          )
+        end
+
+        self.topic_mapping = {}
+
+        config.app_name = Inflecto.underscore(to_s).gsub("/application","")
+
+        load_app!
+        initialize_environment!
+        initialize_router!
+      end
+
+      def config
+        @config ||= Emque::Consuming::Configuration.new
+      end
+
+      def initialize_environment!
+        require_relative File.join(
+          root,
+          "config",
+          "environments",
+          "#{emque_env}.rb"
+        )
+      end
+
+      def initialize_logger
+        Emque::Consuming::Logging.initialize_logger(logfile)
+        Emque::Consuming.logger.level = config.log_level
+        Celluloid.logger = Emque::Consuming.logger
+      end
+
+      def initialize_router!
+        self.router ||= Emque::Consuming::Router.new
+        require_relative File.join(root, "config", "routes.rb")
+      end
+
+      def load_app!
+        app_files = Dir[File.join(root, "app", "**", "*.rb")]
+
+        app_files.each do |app_file|
+          klass = Inflecto.classify(File.basename(app_file, ".rb"))
+          emque_autoload(klass.to_sym, app_file)
+        end
+      end
+
+      def logfile
+        @logfile ||= File.join(root, "log/#{emque_env}.log").tap do |path|
+          directory = File.dirname(path)
+          Dir.mkdir(directory) unless File.exist?(directory)
+        end
+      end
+
+      def logger
+        Emque::Consuming.logger
+      end
+
+      def emque_env
+        config.env_var || ENV["EMQUE_ENV"] || ENV["RACK_ENV"] || "development"
+      end
+    end
+  end
+end

data/lib/emque/consuming/error_tracker.rb

@@ -0,0 +1,39 @@
+require "digest"
+
+module Emque
+  module Consuming
+    class ErrorTracker
+      attr_accessor :occurrences, :limit, :expiration
+
+      def initialize(limit: 5, expiration: 3600)
+        self.limit = limit
+        self.expiration = expiration
+        self.occurrences = {}
+      end
+
+      def notice_error_for(context)
+        occurrences[key_for(context)] = Time.now + expiration
+      end
+
+      def limit_reached?
+        count >= limit
+      end
+
+      def count
+        recent_errors.keys.count
+      end
+
+      private
+
+      def recent_errors
+        occurrences.delete_if do |key, expiration_time|
+          expiration_time < Time.now
+        end
+      end
+
+      def key_for(context)
+        Digest::SHA256.hexdigest(context.to_s)
+      end
+    end
+  end
+end

data/lib/emque/consuming/generators/application.rb

@@ -0,0 +1,95 @@
+require "erb"
+require "fileutils"
+require "optparse"
+
+module Emque
+  module Consuming
+    module Generators
+      class Application
+        IGNORE = [".", ".."]
+
+        def initialize(options, name)
+          self.name = Inflecto.underscore(name)
+          self.options = options
+        end
+
+        def generate
+          context = Class.new(Object) { |obj|
+            def initialize(options, name)
+              @name = Inflecto.camelize(name)
+              @options = options
+            end
+
+            def get_binding; binding; end
+          }.new(options, name).get_binding
+
+          @current_dir = File.realdirpath(Dir.pwd)
+
+          recursively_copy_templates(
+            File.realdirpath(
+              File.join(
+                File.dirname(__FILE__),
+                "..",
+                "..",
+                "..",
+                "templates"
+              )
+            ),
+            [current_dir, name],
+            context
+          )
+        end
+
+        private
+
+        attr_accessor :name, :options
+        attr_reader :current_dir
+
+        def relative_path(path)
+          path.gsub(current_dir, ".")
+        end
+
+        def recursively_copy_templates(path, nesting, context)
+          Dir.entries(path).each do |e|
+            unless IGNORE.include?(e)
+              loc = File.join(path, e)
+
+              if Dir.exists?(loc)
+                new_nest = nesting + [e]
+                create_path = File.join(*new_nest)
+
+                unless Dir.exists?(create_path)
+                  FileUtils.mkdir_p(create_path)
+                  puts "created directory #{relative_path(create_path)}"
+                end
+
+                recursively_copy_templates(loc, new_nest, context)
+              elsif e =~ /\.tt$/
+                filename = File.join(
+                  FileUtils.mkdir_p(File.join(*nesting)).first,
+                  e.gsub(".tt", "")
+                )
+                display = relative_path(filename)
+                overwrite = "Y"
+
+                if File.exists?(filename)
+                  print "#{display} exists, overwrite? (yN) "
+                  overwrite = $stdin.gets
+                end
+
+                if overwrite.upcase.chomp == "Y"
+                  File.open(filename, "w") do |f|
+                    f.write(ERB.new(File.read(loc)).result(context))
+                  end
+                  puts "created file #{display}"
+                else
+                  puts "skipping file #{display}"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/helpers.rb

@@ -0,0 +1,29 @@
+module Emque
+  module Consuming
+    module Helpers
+      private
+
+      def app
+        @app ||=
+          Emque::Consuming.application.instance ||
+          Emque::Consuming.application.new
+      end
+
+      def config
+        Emque::Consuming.application.config
+      end
+
+      def logger
+        Emque::Consuming.application.logger
+      end
+
+      def router
+        Emque::Consuming.application.router
+      end
+
+      def runner
+        Emque::Consuming.runner
+      end
+    end
+  end
+end

data/lib/emque/consuming/logging.rb

@@ -0,0 +1,32 @@
+require "logger"
+
+module Emque
+  module Consuming
+    module Logging
+      class LogFormatter < Logger::Formatter
+        def call(severity, time, progname, msg)
+          "#{time.utc} [#{severity}] #{msg}\n"
+        end
+      end
+
+      def self.initialize_logger(log_target = STDOUT)
+        @logger = Logger.new(log_target)
+        @logger.level = Logger::INFO
+        @logger.formatter = LogFormatter.new
+        @logger
+      end
+
+      def self.logger
+        defined?(@logger) ? @logger : initialize_logger
+      end
+
+      def self.logger=(log)
+        @logger = log || Logger.new("/dev/null")
+      end
+
+      def logger
+        Emque::Consuming::Logging.logger
+      end
+    end
+  end
+end

data/lib/emque/consuming/message.rb

@@ -0,0 +1,22 @@
+require "virtus"
+
+module Emque
+  module Consuming
+    class Message
+      include Virtus.value_object
+
+      values do
+        attribute :offset, Integer
+        attribute :original, Object
+        attribute :partition, Integer
+        attribute :status, Symbol, :default => :processing
+        attribute :topic, Symbol, :default => :unknown
+        attribute :values, Hash, :default => {}
+      end
+
+      def continue?
+        status == :processing
+      end
+    end
+  end
+end

data/lib/emque/consuming/pidfile.rb

@@ -0,0 +1,54 @@
+require "fileutils"
+
+module Emque
+  module Consuming
+    class Pidfile
+      extend Forwardable
+
+      attr_reader :path
+
+      def_delegators :pid, :to_i, :to_s
+
+      def initialize(path)
+        self.path = path
+        ensure_dir_exists
+        self.pid = File.read(path).to_i if File.exists?(path)
+      end
+
+      def running?
+        if pid
+          if pid == 0
+            rm_file
+          else
+            begin
+              Process.getpgid(pid)
+              return true
+            rescue Errno::ESRCH
+              rm_file
+            end
+          end
+        end
+        false
+      end
+
+      def write
+        File.open(path, "w") do |f|
+          f.puts Process.pid
+        end
+      end
+
+      private
+
+      attr_writer :path
+      attr_accessor :pid
+
+      def ensure_dir_exists
+        FileUtils.mkdir_p(File.dirname(path))
+      end
+
+      def rm_file
+        FileUtils.rm_f(path) if File.exists?(path)
+      end
+    end
+  end
+end

data/lib/emque/consuming/router.rb

@@ -0,0 +1,73 @@
+module Emque
+  module Consuming
+    class Router
+      def initialize
+        self.mappings = {}
+      end
+
+      def map(&block)
+        self.instance_eval(&block)
+      end
+
+      def topic(mapping, &block)
+        mapping = Mapping.new(mapping, &block)
+        mappings[mapping.topic.to_sym] ||= []
+        mappings[mapping.topic.to_sym] << mapping
+      end
+
+      def route(topic, type, message)
+        mappings[topic.to_sym].each do |mapping|
+          method = mapping.route(type.to_s)
+
+          if method
+            consumer = mapping.consumer
+
+            consumer.new.consume(method, message)
+          end
+        end
+      end
+
+      def topic_mapping
+        mappings.inject({}) do |hash, (topic, maps)|
+          hash.tap do |h|
+            h[topic] = maps.map(&:consumer)
+          end
+        end
+      end
+
+      def workers(topic)
+        mappings[topic.to_sym].map(&:workers).max
+      end
+
+      private
+
+      attr_accessor :mappings
+
+      class Mapping
+        attr_reader :consumer, :topic, :workers
+
+        def initialize(mapping, &block)
+          self.topic = mapping.keys.first
+          self.workers = mapping.fetch(:workers, 1)
+          self.consumer = mapping.values.first
+          self.mapping = {}
+
+          self.instance_eval(&block)
+        end
+
+        def map(map)
+          mapping.merge!(map)
+        end
+
+        def route(type)
+          mapping[type]
+        end
+
+        private
+
+        attr_accessor :mapping
+        attr_writer :consumer, :topic, :workers
+      end
+    end
+  end
+end