qpush 0.1.0

data/lib/qpush/server/launcher.rb

@@ -0,0 +1,47 @@
+module QPush
+  module Server
+    # Handles the start of the QPush server via command line
+    #
+    class Launcher
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Parses commmand line options and starts the Manager object
+      #
+      def start
+        start_message
+        setup_options
+        boot_manager
+      end
+
+      private
+
+      def start_message
+        Server.log.info('QPush Server starting!')
+        Server.log.info("* Version #{QPush::VERSION}, codename: #{QPush::CODENAME}")
+      end
+
+      # Parses the arguments passed through the command line.
+      #
+      def setup_options
+        parser = OptionParser.new do |o|
+          o.banner = 'Usage: bundle exec bin/QPush [options]'
+
+          o.on('-c', '--config PATH', 'Load PATH for config file') do |arg|
+            load(arg)
+            Server.log.info("* Server config:  #{arg}")
+          end
+
+          o.on('-h', '--help', 'Prints this help') { puts o && exit }
+        end
+        parser.parse!(@argv)
+      end
+
+      def boot_manager
+        manager = Manager.new(QPush.config.manager_options)
+        manager.start
+      end
+    end
+  end
+end

data/lib/qpush/server/logger.rb

@@ -0,0 +1,38 @@
+require 'logger'
+
+module QPush
+  module Server
+    class << self
+      STDOUT.sync = true
+
+      def log
+        @log ||= Log.new
+      end
+    end
+
+    # The Log is a simple wrapper for the Logger. It outputs log info in a
+    # defined manner to the console.
+    #
+    class Log
+      def initialize
+        @log = ::Logger.new(STDOUT)
+        @log.formatter = proc do |_severity, _datetime, _progname, msg|
+          "#{msg}\n"
+        end
+      end
+
+      # Creates a new info log message.
+      #
+      def info(msg)
+        @log.info("[ \e[32mOK\e[0m ] #{msg}")
+      end
+
+      # Creates a new error log message.
+      #
+      def err(msg, action: :no_exit)
+        @log.info("[ \e[31mER\e[0m ] #{msg}")
+        exit 1 if action == :exit
+      end
+    end
+  end
+end

data/lib/qpush/server/manager.rb

@@ -0,0 +1,78 @@
+module QPush
+  module Server
+    # The Manager controls our Worker processes. We use it to instruct each
+    # of them to start and shutdown.
+    #
+    class Manager
+      include ObjectValidator::Validate
+
+      attr_accessor :workers, :options
+      attr_reader :forks
+
+      def initialize(options = {})
+        options.each { |key, value| send("#{key}=", value) }
+        @master = Process.pid
+        @forks = []
+        at_exit { shutdown }
+      end
+
+      # Instantiates new Worker objects, setting them with our options. We
+      # follow up by booting each of our Workers. Our Manager is then put to
+      # sleep so that our Workers can do their thing.
+      #
+      def start
+        validate!
+        start_messages
+        create_workers
+        Process.wait
+      end
+
+      # Shutsdown our Worker processes.
+      #
+      def shutdown
+        unless @forks.empty?
+          @forks.each { |w| Process.kill('SIGTERM', w[:pid].to_i) }
+        end
+        Process.kill('SIGTERM', @master)
+      end
+
+      private
+
+      # Create the specified number of workers and starts them
+      #
+      def create_workers
+        @workers.times do |id|
+          pid = fork { Worker.new(@options.merge(id: id)).start }
+          @forks << { id: id, pid: pid }
+        end
+      end
+
+      # Information about the start process
+      #
+      def start_messages
+        Server.log.info("* Workers: #{@workers}")
+        Server.log.info("* Threads: #{@options[:queue_threads]} queue, #{@options[:perform_threads]} perform, #{@options[:delay_threads]} delay")
+      end
+
+      # Validates our data before starting our Workers. Also instantiates our
+      # connection pool by pinging Redis.
+      #
+      def validate!
+        return if valid?
+        fail ServerError, errors.full_messages.join(' ')
+      end
+    end
+
+    # The ManagerValidator ensures the data for our manager is valid before
+    # attempting to start it.
+    #
+    class ManagerValidator
+      include ObjectValidator::Validator
+
+      validates :redis, with: { proc: proc { QPush.redis.with { |c| c.ping && c.quit } },
+                                msg: 'could not be connected with' }
+      validates :workers, type: Integer, greater_than: 0
+      validates :options, type: Hash
+    end
+  end
+end

data/lib/qpush/server/perform.rb

@@ -0,0 +1,39 @@
+module QPush
+  module Server
+    # The Perform worker runs any jobs that are queued into our Redis server.
+    # It will perform a 'blocking pop' on our job list until one is added.
+    #
+    class Perform
+      def initialize
+        @done = false
+        @lists = QPush.config.perform_lists
+      end
+
+      # Starts our perform process. This will run until instructed to stop.
+      #
+      def start
+        until @done
+          job = retrieve_job
+          job.api.execute if job
+        end
+      end
+
+      # Shutsdown our perform process.
+      #
+      def shutdown
+        @done = true
+      end
+
+      private
+
+      # Performs a 'blocking pop' on our redis job list.
+      #
+      def retrieve_job
+        json = QPush.redis.with { |c| c.brpop(@lists) }
+        Job.new(JSON.parse(json.last))
+      rescue => e
+        raise ServerError, e.message
+      end
+    end
+  end
+end

data/lib/qpush/server/queue.rb

@@ -0,0 +1,39 @@
+module QPush
+  module Server
+    # The Queue worker takes any jobs that are queued into our Redis server,
+    # and moves them to the appropriate list within Redis.
+    # It will perform a 'blocking pop' on our queue list until one is added.
+    #
+    class Queue
+      def initialize
+        @done = false
+      end
+
+      # Starts our queue process. This will run until instructed to stop.
+      #
+      def start
+        until @done
+          job = retrieve_job
+          job.api.setup if job
+        end
+      end
+
+      # Shutsdown our queue process.
+      #
+      def shutdown
+        @done = true
+      end
+
+      private
+
+      # Performs a 'blocking pop' on our redis job list.
+      #
+      def retrieve_job
+        json = QPush.redis.with { |c| c.brpop(QPush.config.queue_namespace) }
+        Job.new(JSON.parse(json.last))
+      rescue => e
+        raise ServerError, e.message
+      end
+    end
+  end
+end

data/lib/qpush/server/worker.rb

@@ -0,0 +1,92 @@
+module QPush
+  module Server
+    # The Worker manages our workers - Queue, Delay, and Perform. Each of these
+    # workers is alloted a number of threads. Each worker object maintains
+    # control of these threads through the aptly named start and shutdown
+    # methods.
+    #
+    class Worker
+      include ObjectValidator::Validate
+
+      attr_accessor :perform_threads, :queue_threads, :delay_threads, :id
+
+      def initialize(options = {})
+        options.each { |key, value| send("#{key}=", value) }
+        @pid = Process.pid
+        @workers = []
+        @threads = []
+        at_exit { shutdown }
+      end
+
+      # Starts our new worker.
+      #
+      def start
+        validate!
+        start_message
+        build_threads
+        start_threads
+      end
+
+      # Shutsdown our worker as well as its threads.
+      #
+      def shutdown
+        shutdown_message
+        @workers.each(&:shutdown)
+        @threads.each(&:exit)
+      end
+
+      private
+
+      # Forks the worker and creates the actual threads (@_threads_real) for
+      # our Queue and Retry objects. We then start them and join them to the
+      # main process.
+      #
+      def start_threads
+        @workers.each do |worker|
+          @threads << Thread.new { worker.start }
+        end
+        @threads.map(&:join)
+      end
+
+      # Instantiates our Queue, Perform, and Delay objects based on the number
+      # of threads specified for each process type. We store these objects as
+      # an array in @threads.
+      #
+      def build_threads
+        @perform_threads.times { @workers << Perform.new }
+        @queue_threads.times { @workers << Queue.new }
+        @delay_threads.times { @workers << Delay.new }
+      end
+
+      # Information about the start process
+      #
+      def start_message
+        Server.log.info("* Worker #{@id} started, pid: #{@pid}")
+      end
+
+      # Information about the shutdown process
+      #
+      def shutdown_message
+        Server.log.info("* Worker #{@id} shutdown, pid: #{@pid}")
+      end
+
+      # Validates our data before starting the worker.
+      #
+      def validate!
+        return if valid?
+        fail ServerError, errors.full_messages.join(' ')
+      end
+    end
+
+    # The WorkerValidator ensures the data for our worker is valid before
+    # attempting to start it.
+    #
+    class WorkerValidator
+      include ObjectValidator::Validator
+
+      validates :perform_threads, type: Integer, greater_than: 0
+      validates :queue_threads, type: Integer, greater_than: 0
+      validates :delay_threads, type: Integer, greater_than: 0
+    end
+  end
+end

data/lib/qpush/version.rb

@@ -0,0 +1,4 @@
+module QPush
+  VERSION = '0.1.0'
+  CODENAME = 'Sun Soaked Salamander'
+end

data/lib/qpush/web.rb

@@ -0,0 +1,7 @@
+# Base
+require 'qpush/base'
+
+# Web
+require 'sinatra/base'
+require 'qpush/web/get'
+require 'qpush/web/server'

data/lib/qpush/web/get.rb

@@ -0,0 +1,60 @@
+module QPush
+  module Web
+    class Get
+      STATS = [:delayed,
+               :queued,
+               :performed,
+               :retries,
+               :success,
+               :failed].freeze
+
+      class << self
+        def stats
+          stats = {}
+          namespace = QPush.config.stats_namespace
+          QPush.redis.with do |conn|
+            STATS.each { |s| stats[s] = conn.get("#{namespace}:#{s}") }
+          end
+          stats
+        end
+
+        def delays(s, e)
+          jobs = Get.all_delays[s, e]
+
+          jobs.map! { |job| JSON.parse(job.first).merge(perform_at: job.last) }
+        end
+
+        def crons(s, e)
+          jobs = Get.all_delays
+
+          jobs.map! do |job|
+            hash = JSON.parse(job.first).merge(perform_at: job.last)
+            hash['cron'].empty? ? next : hash
+          end
+
+          jobs.compact![s, e]
+
+          { total: jobs.count, jobs: jobs }
+        end
+
+        def fails(s, e)
+          jobs = Get.all_delays
+
+          jobs.map! do |job|
+            hash = JSON.parse(job.first).merge(perform_at: job.last)
+            hash['cron'].empty? && hash['total_fail'] > 0 ? hash : next
+          end
+          jobs.compact![s, e]
+
+          { total: jobs.count, jobs: jobs }
+        end
+
+        def all_delays
+          QPush.redis.with do |conn|
+            conn.zrange(QPush.config.delay_namespace, 0, -1, with_scores: true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qpush/web/server.rb

@@ -0,0 +1,25 @@
+module QPush
+  module Web
+    class Server < Sinatra::Base
+      before do
+        content_type :json
+      end
+
+      get '/stats' do
+        Get.stats.to_json
+      end
+
+      get '/delays' do
+        Get.delays(params[:start].to_i, params[:end].to_i).to_json
+      end
+
+      get '/crons' do
+        Get.crons(params[:start].to_i, params[:end].to_i).to_json
+      end
+
+      get '/fails' do
+        Get.fails(params[:start].to_i, params[:end].to_i).to_json
+      end
+    end
+  end
+end

data/lib/qpush/web/server.ru

@@ -0,0 +1,3 @@
+require 'qpush/web'
+
+run QPush::Web::Server