sidekiq-vigil 0.1.0

data/lib/sidekiq_vigil/storage.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SidekiqVigil
+  class Storage
+    class MissingTTL < ArgumentError; end
+    class UnmanagedPersistentKey < ArgumentError; end
+
+    KeyDefinition = Data.define(:pattern, :type, :ttl, :owner)
+    KEY_CATALOG = [
+      KeyDefinition.new(pattern: "stats:{yyyymmddHHMM}", type: "hash", ttl: "8 days", owner: "Reporter"),
+      KeyDefinition.new(pattern: "exec:{queue}", type: "hash", ttl: "1 hour", owner: "Reporter"),
+      KeyDefinition.new(pattern: "alerts", type: "hash", ttl: "managed", owner: "Alert::Manager"),
+      KeyDefinition.new(pattern: "history:{alert_id}", type: "list", ttl: "24 hours", owner: "Alert::Manager"),
+      KeyDefinition.new(pattern: "snapshot", type: "string", ttl: "interval × 4", owner: "Checker"),
+      KeyDefinition.new(pattern: "leader", type: "string", ttl: "interval × 3", owner: "LeaderElection"),
+      KeyDefinition.new(pattern: "mem:{process_id}", type: "string", ttl: "flush interval × 3", owner: "Reporter"),
+      KeyDefinition.new(pattern: "config_digest", type: "hash", ttl: "1 hour", owner: "Reporter"),
+      KeyDefinition.new(pattern: "mute", type: "string", ttl: "requested duration", owner: "Alert::Mute"),
+      KeyDefinition.new(
+        pattern: "check_state:{check-specific-suffix}",
+        type: "string",
+        ttl: "check-specific",
+        owner: "Checks"
+      )
+    ].freeze
+
+    attr_reader :prefix
+
+    def initialize(redis: nil, key_prefix: "default")
+      @redis = redis
+      @prefix = "vigil:#{key_prefix}:"
+    end
+
+    def key(suffix)
+      "#{prefix}#{suffix}"
+    end
+
+    def set(suffix, value, ttl:)
+      require_ttl!(ttl)
+      with_redis { |redis| redis.call("SET", key(suffix), value, "PX", ttl_ms(ttl)) }
+    end
+
+    def get(suffix)
+      with_redis { |redis| redis.call("GET", key(suffix)) }
+    end
+
+    def hash_write(suffix, values, ttl:)
+      require_ttl!(ttl)
+      return if values.empty?
+
+      with_redis do |redis|
+        redis.pipelined do |pipeline|
+          pipeline.call("HSET", key(suffix), *values.flatten)
+          pipeline.call("PEXPIRE", key(suffix), ttl_ms(ttl))
+        end
+      end
+    end
+
+    def hash_increment(suffix, values, ttl:)
+      require_ttl!(ttl)
+      return if values.empty?
+
+      with_redis do |redis|
+        redis.pipelined do |pipeline|
+          values.each do |field, amount|
+            command = amount.is_a?(Integer) ? "HINCRBY" : "HINCRBYFLOAT"
+            pipeline.call(command, key(suffix), field, amount)
+          end
+          pipeline.call("PEXPIRE", key(suffix), ttl_ms(ttl))
+        end
+      end
+    end
+
+    def managed_hash_write(suffix, field, value)
+      raise UnmanagedPersistentKey, "only alerts may be written without a ttl" unless suffix == "alerts"
+
+      with_redis { |redis| redis.call("HSET", key(suffix), field, value) }
+    end
+
+    def hash_get_all(suffix)
+      with_redis { |redis| redis.call("HGETALL", key(suffix)) }
+    end
+
+    def hash_delete(suffix, *fields)
+      return if fields.empty?
+
+      with_redis { |redis| redis.call("HDEL", key(suffix), *fields) }
+    end
+
+    def list_push(suffix, value, ttl:, limit: 30)
+      require_ttl!(ttl)
+      with_redis do |redis|
+        redis.pipelined do |pipeline|
+          pipeline.call("RPUSH", key(suffix), value)
+          pipeline.call("LTRIM", key(suffix), -limit, -1)
+          pipeline.call("PEXPIRE", key(suffix), ttl_ms(ttl))
+        end
+      end
+    end
+
+    def list_range(suffix)
+      with_redis { |redis| redis.call("LRANGE", key(suffix), 0, -1) }
+    end
+
+    def delete(suffix)
+      with_redis { |redis| redis.call("DEL", key(suffix)) }
+    end
+
+    def scan(pattern = "*")
+      with_redis do |redis|
+        cursor = "0"
+        keys = []
+        loop do
+          cursor, batch = redis.call("SCAN", cursor, "MATCH", key(pattern), "COUNT", 100)
+          keys.concat(batch)
+          break if cursor == "0"
+        end
+        keys
+      end
+    end
+
+    def acquire_lock(suffix, token, ttl_ms:)
+      raise MissingTTL, "a positive ttl is required" unless ttl_ms.is_a?(Numeric) && ttl_ms.positive?
+
+      with_redis { |redis| redis.call("SET", key(suffix), token, "NX", "PX", ttl_ms) == "OK" }
+    end
+
+    def eval(script, keys:, argv:)
+      namespaced_keys = keys.map { |item| key(item) }
+      with_redis { |redis| redis.call("EVAL", script, namespaced_keys.length, *namespaced_keys, *argv) }
+    end
+
+    def ping
+      with_redis { |redis| redis.call("PING") }
+    end
+
+    def info(section = nil)
+      with_redis { |redis| parse_info(redis.call("INFO", *Array(section))) }
+    end
+
+    private
+
+    def require_ttl!(ttl)
+      raise MissingTTL, "a positive ttl is required" unless ttl.is_a?(Numeric) && ttl.positive?
+    end
+
+    def ttl_ms(ttl)
+      (ttl * 1_000).ceil
+    end
+
+    def with_redis(&)
+      return yield @redis if @redis
+
+      Sidekiq.redis(&)
+    end
+
+    def parse_info(raw)
+      raw.each_line.filter_map do |line|
+        next if line.start_with?("#") || !line.include?(":")
+
+        line.strip.split(":", 2)
+      end.to_h
+    end
+  end
+end

data/lib/sidekiq_vigil/timezone.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "date"
+
+module SidekiqVigil
+  module Timezone
+    MUTEX = Mutex.new
+
+    module_function
+
+    def local_time(time, zone)
+      return time.getutc if zone == "UTC"
+      return time.getlocal(zone) if zone.match?(/\A[+-]\d{2}:\d{2}\z/)
+
+      MUTEX.synchronize do
+        previous = ENV.fetch("TZ", nil)
+        ENV["TZ"] = zone
+        Time.at(time.to_f).localtime
+      ensure
+        ENV["TZ"] = previous
+      end
+    end
+
+    def same_local_days_ago(time, days, zone)
+      local = local_time(time, zone)
+      date = local.to_date - days
+      return Time.utc(date.year, date.month, date.day, local.hour, local.min, local.sec) if zone == "UTC"
+      return Time.new(date.year, date.month, date.day, local.hour, local.min, local.sec, zone) if zone.match?(/\A[+-]\d{2}:\d{2}\z/)
+
+      MUTEX.synchronize do
+        previous = ENV.fetch("TZ", nil)
+        ENV["TZ"] = zone
+        Time.local(date.year, date.month, date.day, local.hour, local.min, local.sec)
+      ensure
+        ENV["TZ"] = previous
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  VERSION = "0.1.0"
+end

data/lib/sidekiq_vigil.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "logger"
+require "sidekiq"
+require "sidekiq/api"
+
+require_relative "sidekiq_vigil/version"
+
+module SidekiqVigil
+  class Error < StandardError; end
+
+  class << self
+    attr_writer :logger
+
+    def configure
+      yield config
+      config.validate!
+      config
+    end
+
+    def config
+      @config ||= Config.new
+    end
+
+    def collector
+      @collector ||= Collector.new
+    end
+
+    def logger
+      @logger ||= defined?(Sidekiq.logger) ? Sidekiq.logger : Logger.new($stdout)
+    end
+
+    def build_storage(configuration = config)
+      redis = if configuration.redis
+                options = configuration.redis.transform_keys(&:to_sym).compact
+                pool_size = options.delete(:pool_size) || 5
+                pool_timeout = options.delete(:pool_timeout) || 1
+                RedisClient.config(**options).new_pool(size: pool_size, timeout: pool_timeout)
+              end
+      Storage.new(redis:, key_prefix: configuration.key_prefix)
+    end
+
+    def install!
+      return if @installed
+
+      Sidekiq.configure_server do |server|
+        server.server_middleware { |chain| chain.add(Middleware::Server) }
+        server.on(:startup) { start_runtime }
+        server.on(:quiet) { @runtime&.quiet }
+        server.on(:shutdown) { stop_runtime }
+      end
+      @installed = true
+    end
+
+    def start_runtime
+      @runtime ||= Runtime.new(config:, storage: build_storage)
+      @runtime.start
+    end
+
+    def stop_runtime
+      @runtime&.stop
+      @runtime = nil
+    end
+
+    def reset!
+      @config = Config.new
+      @collector = Collector.new
+      @logger = nil
+      @runtime = nil
+    end
+  end
+end
+
+require_relative "sidekiq_vigil/result"
+require_relative "sidekiq_vigil/config"
+require_relative "sidekiq_vigil/storage"
+require_relative "sidekiq_vigil/sidekiq_api"
+require_relative "sidekiq_vigil/timezone"
+require_relative "sidekiq_vigil/check/base"
+require_relative "sidekiq_vigil/check/registry"
+require_relative "sidekiq_vigil/check/queue_latency"
+require_relative "sidekiq_vigil/check/queue_size"
+require_relative "sidekiq_vigil/check/set_size"
+require_relative "sidekiq_vigil/check/process_alive"
+require_relative "sidekiq_vigil/check/utilization"
+require_relative "sidekiq_vigil/check/failure_rate"
+require_relative "sidekiq_vigil/check/stuck_jobs"
+require_relative "sidekiq_vigil/check/memory"
+require_relative "sidekiq_vigil/check/redis_health"
+require_relative "sidekiq_vigil/check/scheduled_backlog"
+require_relative "sidekiq_vigil/check/throughput_anomaly"
+require_relative "sidekiq_vigil/alert/state"
+require_relative "sidekiq_vigil/alert/event"
+require_relative "sidekiq_vigil/alert/cron"
+require_relative "sidekiq_vigil/alert/mute"
+require_relative "sidekiq_vigil/alert/grouping"
+require_relative "sidekiq_vigil/alert/manager"
+require_relative "sidekiq_vigil/notifier/base"
+require_relative "sidekiq_vigil/notifier/log"
+require_relative "sidekiq_vigil/notifier/http_transport"
+require_relative "sidekiq_vigil/notifier/webhook"
+require_relative "sidekiq_vigil/notifier/slack"
+require_relative "sidekiq_vigil/notifier/manager"
+require_relative "sidekiq_vigil/leader_election"
+require_relative "sidekiq_vigil/collector"
+require_relative "sidekiq_vigil/middleware/server"
+require_relative "sidekiq_vigil/reporter"
+require_relative "sidekiq_vigil/checker"
+require_relative "sidekiq_vigil/runtime"
+require_relative "sidekiq_vigil/health_app"
+require_relative "sidekiq_vigil/cli"
+
+SidekiqVigil::BUILT_IN_CHECKS = {
+  queue_latency: SidekiqVigil::Check::QueueLatency,
+  queue_size: SidekiqVigil::Check::QueueSize,
+  retry_set: SidekiqVigil::Check::RetrySet,
+  dead_set: SidekiqVigil::Check::DeadSet,
+  process_alive: SidekiqVigil::Check::ProcessAlive,
+  utilization: SidekiqVigil::Check::Utilization,
+  failure_rate: SidekiqVigil::Check::FailureRate,
+  stuck_jobs: SidekiqVigil::Check::StuckJobs,
+  memory: SidekiqVigil::Check::Memory,
+  redis_health: SidekiqVigil::Check::RedisHealth,
+  scheduled_backlog: SidekiqVigil::Check::ScheduledBacklog,
+  throughput_anomaly: SidekiqVigil::Check::ThroughputAnomaly
+}.freeze
+
+SidekiqVigil.install!

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-vigil
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+description: Monitors Sidekiq health with Redis-backed checks, alert lifecycles, and
+  pluggable notifications.
+email:
+- t.yudai92@gmail.com
+executables:
+- vigil
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- LICENSE.txt
+- README.md
+- Rakefile
+- docs/redis_keys.md
+- docs/webhook_event.schema.json
+- docs/webhook_schema.md
+- exe/vigil
+- lib/generators/sidekiq_vigil/install_generator.rb
+- lib/generators/sidekiq_vigil/templates/sidekiq_vigil.rb
+- lib/sidekiq/vigil.rb
+- lib/sidekiq/vigil/version.rb
+- lib/sidekiq_vigil.rb
+- lib/sidekiq_vigil/alert/cron.rb
+- lib/sidekiq_vigil/alert/event.rb
+- lib/sidekiq_vigil/alert/grouping.rb
+- lib/sidekiq_vigil/alert/manager.rb
+- lib/sidekiq_vigil/alert/mute.rb
+- lib/sidekiq_vigil/alert/state.rb
+- lib/sidekiq_vigil/check/base.rb
+- lib/sidekiq_vigil/check/failure_rate.rb
+- lib/sidekiq_vigil/check/memory.rb
+- lib/sidekiq_vigil/check/process_alive.rb
+- lib/sidekiq_vigil/check/queue_latency.rb
+- lib/sidekiq_vigil/check/queue_size.rb
+- lib/sidekiq_vigil/check/redis_health.rb
+- lib/sidekiq_vigil/check/registry.rb
+- lib/sidekiq_vigil/check/scheduled_backlog.rb
+- lib/sidekiq_vigil/check/set_size.rb
+- lib/sidekiq_vigil/check/stuck_jobs.rb
+- lib/sidekiq_vigil/check/throughput_anomaly.rb
+- lib/sidekiq_vigil/check/utilization.rb
+- lib/sidekiq_vigil/checker.rb
+- lib/sidekiq_vigil/cli.rb
+- lib/sidekiq_vigil/collector.rb
+- lib/sidekiq_vigil/config.rb
+- lib/sidekiq_vigil/health_app.rb
+- lib/sidekiq_vigil/leader_election.rb
+- lib/sidekiq_vigil/middleware/server.rb
+- lib/sidekiq_vigil/notifier/base.rb
+- lib/sidekiq_vigil/notifier/http_transport.rb
+- lib/sidekiq_vigil/notifier/log.rb
+- lib/sidekiq_vigil/notifier/manager.rb
+- lib/sidekiq_vigil/notifier/slack.rb
+- lib/sidekiq_vigil/notifier/webhook.rb
+- lib/sidekiq_vigil/reporter.rb
+- lib/sidekiq_vigil/result.rb
+- lib/sidekiq_vigil/runtime.rb
+- lib/sidekiq_vigil/sidekiq_api.rb
+- lib/sidekiq_vigil/storage.rb
+- lib/sidekiq_vigil/timezone.rb
+- lib/sidekiq_vigil/version.rb
+homepage: https://github.com/ydah/sidekiq-vigil
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/ydah/sidekiq-vigil
+  source_code_uri: https://github.com/ydah/sidekiq-vigil/tree/main
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Battery-included monitoring and alerting for Sidekiq
+test_files: []