qpush 0.1.4 → 0.1.6

data/lib/qpush/server/apis/morgue.rb

@@ -2,10 +2,6 @@ module QPush
   module Server
     module Apis
       class Morgue < Base
-        def initialize(job)
-          @job = job
-        end
-
         def call
           send_to_morgue
         end
@@ -13,9 +9,9 @@ module QPush
         private
 
         def send_to_morgue
-          QPush.redis.with do |conn|
-            conn.hincrby(QPush.keys.stats, 'dead', 1)
-            conn.lpush(QPush.keys.morgue, @job.to_json)
+          Server.redis do |conn|
+            conn.hincrby(Server.keys.stats, 'dead', 1)
+            conn.lpush(Server.keys.morgue, @job.to_json)
           end
         end
       end

data/lib/qpush/server/apis/perform.rb

@@ -2,10 +2,6 @@ module QPush
   module Server
     module Apis
       class Perform < Base
-        def initialize(job)
-          @job = job
-        end
-
         def call
           perform_job
         end
@@ -13,9 +9,9 @@ module QPush
         private
 
         def perform_job
-          QPush.redis.with do |conn|
-            conn.hincrby(QPush.keys.stats, 'performed', 1)
-            conn.lpush("#{QPush.keys.perform}:#{@job.priority}", @job.to_json)
+          Server.redis do |conn|
+            conn.hincrby(Server.keys.stats, 'performed', 1)
+            conn.lpush("#{Server.keys.perform}:#{@job.priority}", @job.to_json)
           end
         end
       end

data/lib/qpush/server/apis/queue.rb

@@ -2,10 +2,6 @@ module QPush
   module Server
     module Apis
       class Queue < Base
-        def initialize(job)
-          @job = job
-        end
-
         def call
           queue_job
         end
@@ -13,9 +9,9 @@ module QPush
         private
 
         def queue_job
-          QPush.redis.with do |conn|
-            conn.hincrby(QPush.keys.stats, 'queued', 1)
-            conn.lpush(QPush.keys.queue, @job.to_json)
+          Server.redis do |conn|
+            conn.hincrby(Server.keys.stats, 'queued', 1)
+            conn.lpush(Server.keys.queue, @job.to_json)
           end
         end
       end

data/lib/qpush/server/apis/setup.rb

@@ -2,10 +2,6 @@ module QPush
   module Server
     module Apis
       class Setup < Base
-        def initialize(job)
-          @job = job
-        end
-
         def call
           invalid_job && return unless @job.valid?
           setup_job
@@ -19,7 +15,7 @@ module QPush
         end
 
         def invalid_job
-          Server.log.err("Job INVALID | #{@job.klass} | #{@job.id} | #{@job.errors.full_messages.join(' ')}")
+          Server.log.err("Worker #{Server.worker.id} | Job INVALID | #{@job.klass} | #{@job.id} | #{@job.errors.full_messages.join(' ')}")
         end
       end
     end

data/lib/qpush/server/apis/success.rb

@@ -2,10 +2,6 @@ module QPush
   module Server
     module Apis
       class Success < Base
-        def initialize(job)
-          @job = job
-        end
-
         def call
           update_job
           stat_increment
@@ -21,13 +17,13 @@ module QPush
         end
 
         def stat_increment
-          QPush.redis.with do |c|
-            c.hincrby(QPush.keys.stats, 'success', 1)
+          Server.redis do |c|
+            c.hincrby(Server.keys.stats, 'success', 1)
           end
         end
 
         def log_success
-          Server.log.info("Job SUCCESS | #{@job.klass} with ID: #{@job.id} | #{@job.run_time}")
+          Server.log.info("Worker #{Server.worker.id} | Job SUCCESS | #{@job.klass} with ID: #{@job.id} | #{@job.run_time}")
         end
 
         def update_history

data/lib/qpush/server/config.rb

@@ -0,0 +1,49 @@
+module QPush
+  module Server
+    include QPush::Base::ConfigHelper
+
+    class << self
+      attr_accessor :worker, :keys
+
+      def config
+        @config ||= Config.new
+      end
+    end
+
+    class WorkerConfig
+      DEFAULTS = {
+        namespace: 'default',
+        priorities: 5,
+        queue_threads: 2,
+        perform_threads: 2,
+        delay_threads: 1 }.freeze
+
+      attr_accessor :perform_threads, :queue_threads, :delay_threads,
+                    :namespace, :priorities
+
+      def initialize(options = {})
+        options = DEFAULTS.merge(options)
+        options.each { |key, value| send("#{key}=", value) }
+      end
+
+      def for_keys
+        { namespace: @namespace, priorities: @priorities }
+      end
+    end
+
+    class Config < QPush::Base::Config
+      SERVER_DEFAULTS = {
+        database_url: ENV['DATABASE_URL'],
+        database_pool: 10,
+        jobs_path: '/jobs',
+        workers: [WorkerConfig.new] }.freeze
+
+      attr_accessor :database_url, :database_pool, :jobs_path, :workers
+
+      def initialize
+        super
+        SERVER_DEFAULTS.each { |key, value| send("#{key}=", value) }
+      end
+    end
+  end
+end

data/lib/qpush/server/database.rb

@@ -7,7 +7,7 @@ module QPush
 
   class Database
     def self.create
-      Sequel.connect(QPush.config.database_url, max_connections: QPush.config.database_pool)
+      Sequel.connect(Server.config.database_url, max_connections: Server.config.database_pool)
     end
   end
 end

data/lib/qpush/server/delay.rb

@@ -14,7 +14,7 @@ module QPush
       #
       def start
         until @done
-          QPush.redis.with do |conn|
+          Server.redis do |conn|
             @conn = conn
             watch_delay { retrieve_delays }
           end
@@ -34,7 +34,7 @@ module QPush
       # If any are found, begin to update them.
       #
       def retrieve_delays
-        delays = @conn.zrangebyscore(QPush.keys.delay, 0, Time.now.to_i)
+        delays = @conn.zrangebyscore(Server.keys.delay, 0, Time.now.to_i)
         delays.any? ? update_delays(delays) : @conn.unwatch
       end
 
@@ -42,7 +42,7 @@ module QPush
       #
       def update_delays(delays)
         @conn.multi do |multi|
-          multi.zrem(QPush.keys.delay, delays)
+          multi.zrem(Server.keys.delay, delays)
           delays.each { |job| perform_job(job) }
         end
       end
@@ -59,7 +59,7 @@ module QPush
       # Performs a watch on our delay list
       #
       def watch_delay
-        @conn.watch(QPush.keys.delay) do
+        @conn.watch(Server.keys.delay) do
           yield if block_given?
         end
       end

data/lib/qpush/server/heartbeat.rb

@@ -1,6 +1,6 @@
 module QPush
   module Server
-    # The Heartbeat worker periodically updates the heart namespace.
+    # The Heartbeat worker periodically updates the heart key.
     # The key is set with an expiry. This helps to indicate if the QPush server
     # is currently active.
     #
@@ -9,11 +9,11 @@ module QPush
         @done = false
       end
 
-      # Starts our perform process. This will run until instructed to stop.
+      # Starts our heartbeat process. This will run until instructed to stop.
       #
       def start
         until @done
-          QPush.redis.with { |c| c.setex(QPush.keys.heart, 30, true) }
+          Server.redis { |c| c.setex(Server.keys.heart, 30, true) }
           sleep 15
         end
       end

data/lib/qpush/server/jobs.rb

@@ -1,5 +1,15 @@
 module QPush
   module Server
+    module JobRegister
+      def included(base)
+        _register_job(base)
+      end
+
+      def _register_job(base)
+        Server.redis { |c| c.sadd(QPush::Base::KEY + ':jobs', base.name) }
+      end
+    end
+
     module JobHelpers
       def mark_success
         @failed = false
@@ -46,12 +56,13 @@ module QPush
       end
     end
 
-    class Job < QPush::Job::Base
+    class Job < QPush::Base::Job
       extend Forwardable
 
       include QPush::Server::JobHelpers
       include ObjectValidator::Validate
 
+
       def initialize(options)
         super
         @api = ApiWrapper.new(self)

data/lib/qpush/server/launcher.rb

@@ -27,7 +27,7 @@ module QPush
       #
       def setup_options
         parser = OptionParser.new do |o|
-          o.banner = 'Usage: bundle exec bin/QPush [options]'
+          o.banner = 'Usage: bundle exec qpush-server [options]'
 
           o.on('-c', '--config PATH', 'Load PATH for config file') do |arg|
             load(arg)
@@ -42,11 +42,11 @@ module QPush
       # Requires all base jobs as well as user jobs.
       #
       def setup_jobs
-        Loader.call
+        JobLoader.call
       end
 
       def boot_manager
-        manager = Manager.new(QPush.config.manager_options)
+        manager = Manager.new(Server.config.workers)
         manager.start
       end
     end

data/lib/qpush/server/loader.rb

@@ -3,18 +3,18 @@ module QPush
     # The Loader will 'require' all jobs within the users job folder.
     # The job folder is specified in the config.
     #
-    class Loader
+    class JobLoader
       # Provides a shortend caller.
       #
       def self.call
-        jobs = Loader.new
-        jobs.call
+        loader = new
+        loader.call
       end
 
       # Entrypoint to load all jobs.
       #
       def call
-        remove_old
+        flush_jobs
         load_jobs
       end
 
@@ -22,14 +22,14 @@ module QPush
 
       # Removes old jobs from the redis job list.
       #
-      def remove_old
-        QPush.redis.with { |c| c.del(QPush.keys.jobs) }
+      def flush_jobs
+        Server.redis { |c| c.del(QPush::Base::KEY + ':jobs') }
       end
 
       # Requires user jobs that are specified from the config.
       #
       def load_jobs
-        Dir[Dir.pwd + "#{QPush.config.jobs_path}/**/*.rb"].each do |file|
+        Dir[Dir.pwd + "#{Server.config.jobs_path}/**/*.rb"].each do |file|
           require file
         end
       end

data/lib/qpush/server/manager.rb

@@ -6,11 +6,11 @@ module QPush
     class Manager
       include ObjectValidator::Validate
 
-      attr_accessor :workers, :options
+      attr_accessor :configs
       attr_reader :forks
 
-      def initialize(options = {})
-        options.each { |key, value| send("#{key}=", value) }
+      def initialize(configs)
+        @configs = configs
         @master = Process.pid
         @forks = []
         at_exit { shutdown }
@@ -23,6 +23,7 @@ module QPush
       def start
         validate!
         start_messages
+        flush_spaces
         create_workers
         Process.wait
       end
@@ -31,8 +32,9 @@ module QPush
       #
       def shutdown
         unless @forks.empty?
-          @forks.each { |w| Process.kill('SIGTERM', w[:pid].to_i) }
+          @forks.each { |w| Process.kill('QUIT', w[:pid].to_i) }
         end
+        Process.waitall
         Process.kill('SIGTERM', @master)
       end
 
@@ -41,8 +43,8 @@ module QPush
       # Create the specified number of workers and starts them
       #
       def create_workers
-        @workers.times do |id|
-          pid = fork { Worker.new(@options.merge(id: id)).start }
+        @configs.each_with_index do |config, id|
+          pid = fork { Worker.new(id, config).start }
           @forks << { id: id, pid: pid }
         end
       end
@@ -50,8 +52,7 @@ module QPush
       # Information about the start process
       #
       def start_messages
-        Server.log.info("* Workers: #{@workers}")
-        Server.log.info("* Threads: #{@options[:queue_threads]} queue, #{@options[:perform_threads]} perform, #{@options[:delay_threads]} delay")
+        Server.log.info("* Worker count: #{@configs.count}")
       end
 
       # Validates our data before starting our Workers. Also instantiates our
@@ -61,6 +62,13 @@ module QPush
         return if valid?
         fail ServerError, errors.full_messages.join(' ')
       end
+
+      # Removes the list of namespaces used by our server from Redis. This
+      # prepares it for the new list that will be created by our workers.
+      #
+      def flush_spaces
+        Server.redis { |c| c.del(QPush::Base::KEY + ':namespaces') }
+      end
     end
 
     # The ManagerValidator ensures the data for our manager is valid before
@@ -69,10 +77,12 @@ module QPush
     class ManagerValidator
       include ObjectValidator::Validator
 
-      validates :redis, with: { proc: proc { QPush.redis.with { |c| c.ping && c.quit } },
+      validates :redis, with: { proc: proc { Server.redis { |c| c.ping && c.quit } },
                                 msg: 'could not be connected with' }
-      validates :workers, type: Integer, greater_than: 0
-      validates :options, type: Hash
+      validates :configs, with: { proc: proc { |m| m.configs.count > 0 },
+                                  msg: 'were not defined' }
+      validates :configs, with: { proc: proc { |m| m.configs.each { |c| c.is_a?(WorkerConfig) } },
+                                  msg: 'are not valid WorkerConfig objects' }
     end
   end
 end

data/lib/qpush/server/perform.rb

@@ -6,7 +6,6 @@ module QPush
     class Perform
       def initialize
         @done = false
-        @lists = QPush.keys.perform_lists
       end
 
       # Starts our perform process. This will run until instructed to stop.
@@ -29,7 +28,7 @@ module QPush
       # Performs a 'blocking pop' on our redis job list.
       #
       def retrieve_job
-        json = QPush.redis.with { |c| c.brpop(@lists) }
+        json = Server.redis { |c| c.brpop(Server.keys.perform_list) }
         Job.new(JSON.parse(json.last))
       rescue => e
         raise ServerError, e.message

data/lib/qpush/server/queue.rb

@@ -29,7 +29,7 @@ module QPush
       # Performs a 'blocking pop' on our redis job list.
       #
       def retrieve_job
-        json = QPush.redis.with { |c| c.brpop(QPush.keys.queue) }
+        json = Server.redis { |c| c.brpop(Server.keys.queue) }
         Job.new(JSON.parse(json.last))
       rescue => e
         raise ServerError, e.message