qpush 0.1.0

data/lib/qpush/job.rb

@@ -0,0 +1,72 @@
+module QPush
+  class << self
+    def job(options)
+      job = Job::Wrapper.new(options)
+      job.queue
+    end
+  end
+
+  module Job
+    class << self
+      def included(base)
+        base.extend(ClassMethods)
+      end
+    end
+
+    module ClassMethods
+      def queue(options)
+        QPush.job(options.merge(klass: name))
+      end
+    end
+
+    module Base
+      def to_json
+        { klass: @klass,
+          id: @id,
+          priority: @priority,
+          created_at: @created_at,
+          start_at: @start_at,
+          cron: @cron,
+          retry_max: @retry_max,
+          total_fail: @total_fail,
+          total_success: @total_success,
+          args: @args }.to_json
+      end
+
+      private
+
+      def defaults
+        { id: SecureRandom.urlsafe_base64,
+          args: {},
+          priority: 3,
+          created_at: Time.now.to_i,
+          start_at: Time.now.to_i - 1,
+          cron: '',
+          retry_max: 10,
+          total_fail: 0,
+          total_success: 0
+        }
+      end
+    end
+
+    class Wrapper
+      include QPush::Job::Base
+
+      attr_accessor :klass, :id, :priority, :created_at, :start_at,
+                    :cron, :retry_max, :total_success, :total_fail,
+                    :args
+
+      def initialize(options = {})
+        options = defaults.merge(options)
+        options.each { |key, value| send("#{key}=", value) }
+      end
+
+      def queue
+        QPush.redis.with do |conn|
+          conn.incr("#{QPush.config.stats_namespace}:queued")
+          conn.lpush("#{QPush.config.queue_namespace}", to_json)
+        end
+      end
+    end
+  end
+end

data/lib/qpush/jobs/test_job.rb

@@ -0,0 +1,5 @@
+class TestJob
+  def call
+    puts 'hello'
+  end
+end

data/lib/qpush/redis.rb

@@ -0,0 +1,17 @@
+module QPush
+  class << self
+    attr_reader :redis_pool
+
+    def redis
+      @redis_pool ||= RedisPool.create
+    end
+  end
+
+  class RedisPool
+    def self.create
+      ::ConnectionPool.new(size: QPush.config.redis_pool) do
+        ::Redis.new(url: QPush.config.redis_url)
+      end
+    end
+  end
+end

data/lib/qpush/server.rb

@@ -0,0 +1,18 @@
+# Base
+require 'qpush/base'
+
+# Server
+require 'sequel'
+require 'object_validator'
+require 'parse-cron'
+require 'qpush/server/database'
+require 'qpush/server/delay'
+require 'qpush/server/errors'
+require 'qpush/server/execute'
+require 'qpush/server/jobs'
+require 'qpush/server/launcher'
+require 'qpush/server/logger'
+require 'qpush/server/manager'
+require 'qpush/server/perform'
+require 'qpush/server/queue'
+require 'qpush/server/worker'

data/lib/qpush/server/database.rb

@@ -0,0 +1,13 @@
+module QPush
+  class << self
+    def db
+      @db ||= Database.create
+    end
+  end
+
+  class Database
+    def self.create
+      Sequel.connect(QPush.config.database_url, max_connections: QPush.config.database_pool)
+    end
+  end
+end

data/lib/qpush/server/delay.rb

@@ -0,0 +1,68 @@
+module QPush
+  module Server
+    # The Delay worker requeues any jobs that have been delayed on our Redis
+    # server. Delayed jobs are pulled by a 'zrangebyscore', with the score
+    # representing the time the job should be performed.
+    #
+    class Delay
+      def initialize
+        @done = false
+        @conn = nil
+      end
+
+      # Starts our delay process. This will run until instructed to stop.
+      #
+      def start
+        until @done
+          QPush.redis.with do |conn|
+            @conn = conn
+            watch_delay { retrieve_delays }
+          end
+          sleep 2
+        end
+      end
+
+      # Shutsdown our dleay process.
+      #
+      def shutdown
+        @done = true
+      end
+
+      private
+
+      # Retrieves delayed jobs based on the time they should be performed.
+      # If any are found, begin to update them.
+      #
+      def retrieve_delays
+        delays = @conn.zrangebyscore(QPush.config.delay_namespace, 0, Time.now.to_i)
+        delays.any? ? update_delays(delays) : @conn.unwatch
+      end
+
+      # Removes jobs that have been retrieved and sets them up to be performed.
+      #
+      def update_delays(delays)
+        @conn.multi do |multi|
+          multi.zrem(QPush.config.delay_namespace, delays)
+          delays.each { |job| perform_job(job) }
+        end
+      end
+
+      # Add a delayed job to the appropriate perform list.
+      #
+      def perform_job(json)
+        job = Job.new(JSON.parse(json))
+        job.api.perform
+      rescue => e
+        raise ServerError, e.message
+      end
+
+      # Performs a watch on our delay list
+      #
+      def watch_delay
+        @conn.watch(QPush.config.delay_namespace) do
+          yield if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/qpush/server/errors.rb

@@ -0,0 +1,18 @@
+module QPush
+  class ServerError < StandardError
+    def initialize(msg = nil)
+      @message = msg
+      log_error
+    end
+
+    def message
+      "The following error occured: #{@message}"
+    end
+
+    private
+
+    def log_error
+      Server.log.err("Server Error - #{@message}")
+    end
+  end
+end

data/lib/qpush/server/execute.rb

@@ -0,0 +1,92 @@
+module QPush
+  module Server
+    class Execute
+      def initialize(job)
+        @job = job
+      end
+
+      def call
+        measure_run_time { job_object.call }
+        ExecutionSuccess.call(@job)
+      rescue => e
+        ExecutionFail.call(@job, e)
+      end
+
+      private
+
+      def measure_run_time
+        start = Time.now
+        yield
+        finish = Time.now
+        @job.run_time = "#{((finish - start) * 1000.0).round(3)} ms"
+      end
+
+      def job_object
+        klass = Object.const_get(@job.klass)
+        @job.args.empty? ? klass.new : klass.new(@job.args)
+      end
+    end
+
+    class ExecutionFail
+      def self.call(*args)
+        failed = ExecutionFail.new(*args)
+        failed.call
+      end
+
+      def initialize(job, error)
+        @job = job
+        @error = error
+      end
+
+      def call
+        @job.bump_fail
+        @job.api.retry if @job.retry_job?
+        stat_increment
+        log_error
+      end
+
+      private
+
+      def stat_increment
+        QPush.redis.with do |c|
+          c.incr("#{QPush.config.stats_namespace}:dead") if @job.dead_job?
+          c.incr("#{QPush.config.stats_namespace}:failed")
+        end
+      end
+
+      def log_error
+        Server.log.err("Job FAILED | #{@job.klass} | #{@job.id} | #{@error.message}")
+      end
+    end
+
+    class ExecutionSuccess
+      def self.call(*args)
+        success = ExecutionSuccess.new(*args)
+        success.call
+      end
+
+      def initialize(job)
+        @job = job
+      end
+
+      def call
+        @job.bump_success
+        @job.api.delay if @job.delay_job?
+        stat_increment
+        log_success
+      end
+
+      private
+
+      def stat_increment
+        QPush.redis.with do |c|
+          c.incr("#{QPush.config.stats_namespace}:success")
+        end
+      end
+
+      def log_success
+        Server.log.info("Job SUCCESS | #{@job.klass} with ID: #{@job.id} | #{@job.run_time}")
+      end
+    end
+  end
+end

data/lib/qpush/server/jobs.rb

@@ -0,0 +1,139 @@
+module QPush
+  module Server
+    module JobHelpers
+      def bump_success
+        @total_success += 1
+      end
+
+      def bump_fail
+        @total_fail += 1
+      end
+
+      def retry_job?
+        @retry_max > @total_fail
+      end
+
+      def perform_job?
+        @start_at < Time.now.to_i && @cron.empty?
+      end
+
+      def delay_job?
+        (@start_at > Time.now.to_i && @cron.empty?) || cron_job?
+      end
+
+      def cron_job?
+        @start_at < Time.now.to_i && !@cron.empty?
+      end
+
+      def dead_job?
+        @total_fail >= @retry_max
+      end
+
+      def cron_at
+        CronParser.new(@cron).next(Time.now).to_i
+      rescue => e
+        raise ServerError, e.message
+      end
+
+      def delay_until
+        @cron.empty? ? @start_at : cron_at
+      end
+
+      def retry_at
+        Time.now.to_i + ((@total_fail**4) + 15 + (rand(30) * (@total_fail + 1)))
+      end
+    end
+
+    class Job
+      include QPush::Job::Base
+      include QPush::Server::JobHelpers
+      include ObjectValidator::Validate
+
+      attr_accessor :klass, :id, :priority, :created_at, :start_at,
+                    :cron, :retry_max, :total_success, :total_fail,
+                    :run_time
+      attr_reader :args, :api
+
+      def initialize(options = {})
+        options = defaults.merge(options)
+        options.each { |key, value| send("#{key}=", value) }
+        @api = JobApi.new(self)
+      end
+
+      def args=(args)
+        @args =
+          if args.is_a?(String) then JSON.parse(args)
+          else args
+          end
+      rescue JSON::ParserError
+        @args = nil
+      end
+    end
+
+    class JobValidator
+      include ObjectValidator::Validator
+
+      validates :klass,
+                with: { proc: proc { |j| Object.const_defined?(j.klass) },
+                        msg: 'has not been defined' }
+      validates :cron,
+                with: { proc: proc { |j| j.cron.empty? ? true : CronParser.new(j.cron) },
+                        msg: 'is not a valid expression' }
+      validates :id, type: String
+      validates :args, type: Hash
+      validates :created_at, type: Integer
+      validates :start_at, type: Integer
+      validates :retry_max, type: Integer
+      validates :total_fail, type: Integer
+      validates :total_success, type: Integer
+    end
+
+    class JobApi
+      def initialize(job)
+        @job = job
+        @config = QPush.config
+      end
+
+      def delay
+        QPush.redis.with do |conn|
+          conn.incr("#{@config.stats_namespace}:delayed")
+          conn.zadd(@config.delay_namespace, @job.delay_until, @job.to_json)
+        end
+      end
+
+      def queue
+        QPush.redis.with do |conn|
+          conn.incr("#{@config.stats_namespace}:queued")
+          conn.lpush("#{@config.queue_namespace}", @job.to_json)
+        end
+      end
+
+      def execute
+        execute = Execute.new(@job)
+        execute.call
+      end
+
+      def perform
+        QPush.redis.with do |conn|
+          conn.incr("#{@config.stats_namespace}:performed")
+          conn.lpush("#{@config.perform_namespace}:#{@job.priority}", @job.to_json)
+        end
+      end
+
+      def retry
+        QPush.redis.with do |conn|
+          conn.incr("#{@config.stats_namespace}:retries")
+          conn.zadd(@config.delay_namespace, @job.retry_at, @job.to_json)
+        end
+      end
+
+      def setup
+        fail unless @job.valid?
+        perform if @job.perform_job?
+        delay if @job.delay_job?
+      rescue
+        raise ServerError, 'Invalid job: ' + @job.errors.full_messages.join(' ')
+      end
+    end
+  end
+end