resque-alive 0.1.0

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/tilt

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'tilt' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("tilt", "tilt")

data/lib/resque/alive.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "resque/plugins/alive"

data/lib/resque/plugins/alive.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "resque"
+require "resque/plugins/alive/config"
+require "resque/plugins/alive/heartbeat"
+require "resque/plugins/alive/server"
+require "resque/plugins/alive/version"
+
+module Resque
+  module Plugins
+    module Alive
+      module Redis
+        TTL_EXPIRED = -2
+      end
+
+      def self.procline(string)
+        $0 = "resque-alive-#{Resque::Plugins::Alive::VERSION}: #{string}"
+        logger.send(:debug, $0)
+      end
+
+      def self.start
+        append_resque_alive_heartbeat_queue
+        Resque.before_first_fork do
+          Resque::Plugins::Alive.tap do |resque_alive|
+            procline("starting")
+            logger.info(banner)
+            register_current_instance
+            store_alive_key
+
+            procline("registering heartbeat")
+            Resque.enqueue(
+              Heartbeat,
+              hostname
+            )
+
+            procline("initializing webserver")
+            @server_pid = fork do
+              procline("listening on port: #{config.port} at path: #{config.path}")
+              resque_alive::Server.run!
+            end
+
+            logger.info(successful_startup_text)
+
+            # TODO: worker_exit is, an as yet unreleased feature of
+            # resque. If the version of resque used by the host
+            # application doesn't yet support worker_exit, register an
+            # Kernel#at_exit hook to gracefully shutdown resque-alive.
+            unless Resque.respond_to?(:worker_exit)
+              at_exit do
+                Resque::Plugins::Alive.shutdown
+              end
+            end
+          end
+
+          # TODO: worker_exit is, an as yet unreleased feature of
+          # resque, but should be the preferred method of triggering a
+          # graceful shutdown of resque-alive.
+          #
+          # https://github.com/resque/resque/blob/master/HISTORY.md#unreleased
+          if Resque.respond_to?(:worker_exit)
+            Resque.worker_exit do
+              Resque::Plugins::Alive.shutdown
+            end
+          end
+        end
+
+        Resque.before_pause do
+          Resque::Plugins::Alive.unregister_current_instance
+        end
+      end
+
+      def self.shutdown
+        procline("shutting down webserver #{@server_pid}")
+        Process.kill('TERM', @server_pid) unless @server_pid.nil?
+        Process.wait(@server_pid) unless @server_pid.nil?
+
+        procline("unregistering resque_alive")
+        Resque::Plugins::Alive.unregister_current_instance
+
+        procline("shutting down...")
+      end
+
+      QUEUE_ENV_VARS = %w(QUEUE QUEUES)
+      def self.append_resque_alive_heartbeat_queue
+        QUEUE_ENV_VARS.each do |env_var|
+          if ENV[env_var]
+            ENV[env_var] = [ENV[env_var], current_queue].join(",")
+          end
+        end
+      end
+
+      def self.config
+        @config ||= Config.instance
+      end
+
+      def self.setup
+        yield(config)
+      end
+
+      def self.redis
+        Resque.redis { |r| r }
+      end
+
+      def self.current_liveness_key
+        "#{config.liveness_key}::#{hostname}"
+      end
+
+      def self.hostname
+        config.hostname
+      end
+
+      def self.store_alive_key
+        redis.set(
+          current_liveness_key,
+          Time.now.to_i,
+          ex: config.time_to_live.to_i
+        )
+      end
+
+      def self.alive?
+        redis.ttl(current_liveness_key) != Redis::TTL_EXPIRED
+      end
+
+      def self.registered_instances
+        redis.keys("#{config.registered_instance_key}::*")
+      end
+
+      def self.register_current_instance
+        register_instance(current_instance_register_key)
+      end
+
+      def self.current_instance_register_key
+        "#{config.registered_instance_key}::#{hostname}"
+      end
+
+
+      def self.register_instance(instance_name)
+        redis.set(
+          instance_name,
+          Time.now.to_i,
+          ex: config.registration_ttl.to_i
+        )
+      end
+
+      def self.unregister_current_instance
+        # Delete any pending jobs for this instance
+        logger.info(shutdown_info)
+        purge_pending_jobs
+        redis.del(current_instance_register_key)
+      end
+
+      def self.purge_pending_jobs
+        logger.info("[Resque::Plugins::Alive] Begin purging pending jobs for queue #{current_queue}")
+        pending_heartbeat_jobs_count = Resque::Job.destroy(current_queue, Heartbeat)
+        logger.info("[Resque::Plugins::Alive] Purged #{pending_heartbeat_jobs_count} pending for #{hostname}")
+        logger.info("[Resque::Plugins::Alive] Removing queue #{current_queue}")
+        Resque.remove_queue(current_queue)
+        logger.info("[Resque::Plugins::Alive] Finished purging pending jobs for queue #{current_queue}")
+      end
+
+      def self.logger
+        Resque.logger
+      end
+
+      def self.banner
+        <<~BANNER
+          =================== Resque::Plugins::Alive =================
+          Hostname: #{hostname}
+          Liveness key: #{current_liveness_key}
+          Port: #{config.port}
+          Time to live: #{config.time_to_live}s
+          Current instance register key: #{current_instance_register_key}
+          Worker running on queue: #{@queue}
+          starting ...
+        BANNER
+      end
+
+      def self.shutdown_info
+        <<~BANNER
+          =================== Shutting down Resque::Plugins::Alive =================
+          Hostname: #{hostname}
+          Liveness key: #{current_liveness_key}
+          Current instance register key: #{current_instance_register_key}
+        BANNER
+      end
+
+      def self.current_queue
+        Heartbeat.current_queue
+      end
+
+      def self.successful_startup_text
+        <<~BANNER
+          Registered instances:
+          - #{registered_instances.join("\n\s\s- ")}
+          =================== Resque::Plugins::Alive Ready! =================
+        BANNER
+      end
+
+      def self.enabled?
+        config.enabled
+      end
+    end
+  end
+end
+
+Resque::Plugins::Alive.start if Resque::Plugins::Alive.enabled?

data/lib/resque/plugins/alive/config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Resque
+  module Plugins
+    module Alive
+      class Config
+        include Singleton
+
+        def initialize
+          set_defaults
+        end
+
+        def set_defaults
+          self.port = ENV["RESQUE_ALIVE_PORT"] || 7433
+          self.path = ENV["RESQUE_ALIVE_PATH"] || "/"
+          self.liveness_key = "RESQUE::LIVENESS_PROBE_TIMESTAMP"
+          self.time_to_live = 10 * 60
+          self.callback = proc {}
+          self.registered_instance_key = "RESQUE_REGISTERED_INSTANCE"
+          self.queue_prefix = :resque_alive
+          self.server = ENV["RESQUE_ALIVE_SERVER"] || "webrick"
+          self.hostname = ENV["HOSTNAME"] || "HOSTNAME_NOT_SET"
+          self.enabled = !ENV["RESQUE_ALIVE_DISABLED"]
+        end
+
+        def enabled?
+          enabled
+        end
+
+        def registration_ttl
+          @registration_ttl || time_to_live + 60
+        end
+
+        attr_accessor(
+          :callback,
+          :enabled,
+          :hostname,
+          :liveness_key,
+          :path,
+          :port,
+          :queue_prefix,
+          :registered_instance_key,
+          :server,
+          :time_to_live,
+        )
+      end
+    end
+  end
+end

data/lib/resque/plugins/alive/heartbeat.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "resque-scheduler"
+
+module Resque
+  module Plugins
+    module Alive
+      class Heartbeat
+        # TODO: For the case where multiple workers exist on the same
+        # host, maybe this needs to know the PID of the worker process
+        # and namespace the queue to that pid?
+        def self.perform(_hostname = config.hostname)
+          ping
+          schedule_next_heartbeat
+        end
+
+        def self.ping
+          Alive.store_alive_key
+          Alive.register_current_instance
+
+          begin
+            config.callback.call
+          rescue StandardError
+            nil
+          end
+        end
+
+        def self.schedule_next_heartbeat
+          Resque.enqueue_in_with_queue(
+            current_queue,
+            inside_ttl_window,
+            self.name,
+            current_hostname
+          )
+        end
+
+        def self.inside_ttl_window
+          config.time_to_live / 2
+        end
+
+        def self.current_hostname
+          config.hostname
+        end
+
+        def self.config
+          Config.instance
+        end
+
+        def self.current_queue
+          "#{config.queue_prefix}-#{current_hostname}"
+        end
+
+        @queue = current_queue
+      end
+    end
+  end
+end

data/lib/resque/plugins/alive/server.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "rack"
+
+module Resque
+  module Plugins
+    module Alive
+      class Server
+        class << self
+          def run!
+            handler = Rack::Handler.get(server)
+
+            Signal.trap("TERM") { handler.shutdown }
+
+            handler.run(self, Port: port, Host: '0.0.0.0')
+          end
+
+          def port
+            config.port
+          end
+
+          def path
+            config.path
+          end
+
+          def server
+            config.server
+          end
+
+          def config
+            Alive.config
+          end
+
+          def call(env)
+            if Rack::Request.new(env).path != path
+              [404, {}, ["Received unknown path"]]
+            elsif Alive.alive?
+              [200, {}, ["Alive key is present"]]
+            else
+              response = "Alive key is absent"
+              Alive.logger.error(response)
+              [404, {}, [response]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/alive/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Resque
+  module Plugins
+    module Alive
+      VERSION = "0.1.0"
+    end
+  end
+end

data/resque-alive.gemspec

@@ -0,0 +1,52 @@
+require_relative "lib/resque/plugins/alive/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "resque-alive"
+  spec.version       = Resque::Plugins::Alive::VERSION
+  spec.authors       = ["Aaron Kuehler"]
+  spec.email         = ["aaron.kuehler@gmail.com"]
+
+  spec.summary       = %q{Adds a Kubernetes Liveness probe to Resque}
+  spec.description   = <<~EOD
+
+    resque-alive adds a Kubernetes Liveness probe to a Resque instance.
+
+    How?
+
+    resque-alive provides a small rack application which
+    exposes HTTP endpoint to return the "Aliveness" of the Resque
+    instance. Aliveness is determined by the presence of an
+    auto-expiring key. resque-alive schedules a "heartbeat"
+    job to periodically refresh the expiring key - in the event the
+    Resque instance can"t process the job, the key expires and the
+    instance is marked as unhealthy.
+  EOD
+
+  spec.homepage      = "https://github.com/indiebrain/resque-alive"
+  spec.license       = "GPL-3.0"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/indiebrain/resque-alive"
+  spec.metadata["changelog_uri"] = "https://github.com/indiebrain/resque-alive/blob/master/changelog.txt"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", ">= 1.16"
+  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "mock_redis"
+  spec.add_development_dependency "rack-test"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "resque_spec", "~> 0.18.1"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency "resque"
+  spec.add_dependency "resque-scheduler"
+end