schked 1.4.0 → 2.0.0

data/lib/schked/database_connection.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Schked
+  # Auto-detects which +Schked::Adapters::*+ class to use for the
+  # database-backed coordination store. Returns a concrete +JobRunStore+
+  # instance — the caller should not need to know which backend is in play.
+  #
+  # Resolution order:
+  # 1. An explicit connection/pool if provided.
+  # 2. +ActiveRecord::Base.connection_pool+ when ActiveRecord is loaded
+  #    and configured (the pool, not a single connection, so the adapter
+  #    can check out per operation).
+  # 3. Sequel's first available +Sequel::DATABASES+ database.
+  # 4. Otherwise raises a clear +NotFoundError+ telling the operator to
+  #    set +database_connection+ explicitly.
+  module DatabaseConnection
+    class NotFoundError < StandardError; end
+
+    module_function
+
+    def detect(connection: nil, logger: Logger.new($stdout))
+      raw = connection || auto_detect_connection
+      wrap(raw, logger: logger)
+    end
+
+    def wrap(raw, logger: Logger.new($stdout))
+      case raw
+      when Adapters::Sequel, Adapters::ActiveRecord
+        raw
+      else
+        build_for(raw, logger: logger)
+      end
+    end
+
+    def build_for(raw, logger: Logger.new($stdout))
+      if defined?(::Sequel::Database) && raw.is_a?(::Sequel::Database)
+        Adapters::Sequel.new(raw, logger: logger)
+      elsif raw.respond_to?(:with_connection)
+        # An ActiveRecord connection pool.
+        Adapters::ActiveRecord.new(raw, logger: logger)
+      elsif raw.respond_to?(:pool) && raw.respond_to?(:adapter_name)
+        # A concrete ActiveRecord adapter connection — normalize to its pool.
+        Adapters::ActiveRecord.new(raw.pool, logger: logger)
+      elsif raw.respond_to?(:claim) && raw.respond_to?(:cleanup)
+        # Already a store: a pre-built adapter or a custom object.
+        raw
+      else
+        raise NotFoundError,
+          "Schked could not detect an adapter for: #{raw.class}. " \
+          "Provide a Sequel::Database, an ActiveRecord connection pool, " \
+          "or an ActiveRecord connection (PostgreSQL, Mysql2, or Trilogy)."
+      end
+    end
+
+    def auto_detect_connection
+      if defined?(ActiveRecord) && ActiveRecord.const_defined?(:Base)
+        begin
+          return ActiveRecord::Base.connection_pool
+        rescue
+          # ActiveRecord is loaded but not configured (or the pool cannot
+          # be resolved) — fall through to Sequel detection below.
+        end
+      end
+
+      if defined?(Sequel) && Sequel.respond_to?(:DATABASES) && Sequel::DATABASES.any?
+        return Sequel::DATABASES.first
+      end
+
+      raise NotFoundError,
+        "Schked could not detect a database connection for the database-backed " \
+        "job run store. Load ActiveRecord or Sequel, or set `Schked.config.database_connection` " \
+        "explicitly."
+    end
+  end
+end

data/lib/schked/job_run_store.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Schked
+  # Abstract interface for the per-job coordination store used by the
+  # deduplication mode. A store records which `(job_name, window_start)` pairs
+  # have been claimed, so two scheduler instances cannot execute the same job
+  # in the same interval.
+  #
+  # Two built-in backends are provided: Redis (see +RedisJobRunStore+) and
+  # database (see +DatabaseJobRunStore+). An operator may also supply any
+  # custom object that responds to +#claim+ and +#cleanup+.
+  module JobRunStore
+    # Atomically records that +job_name+ was claimed for the interval starting
+    # at +window_start+ (a Time/Integer epoch).
+    #
+    # Returns +true+ if this caller won the claim (the job may run) and +false+
+    # if the interval is already claimed by another instance (the job must be
+    # skipped). Raises +ArgumentError+ for invalid arguments.
+    def claim(job_name, window_start)
+      raise NotImplementedError, "#{self.class} must implement #claim(job_name, window_start)"
+    end
+
+    # Removes records whose window_start is older than +older_than+. The
+    # Redis backend relies on native TTL and provides a no-op; the database
+    # backend deletes matching rows.
+    def cleanup(older_than)
+      raise NotImplementedError, "#{self.class} must implement #cleanup(older_than)"
+    end
+  end
+end

data/lib/schked/liveness_probe.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "socket"
+require "ipaddr"
+
+module Schked
+  class LivenessProbeConfig
+    attr_reader :enabled, :bind, :port, :path, :heartbeat_interval, :heartbeat_threshold
+
+    DEFAULTS = {
+      enabled: false,
+      bind: "0.0.0.0",
+      port: 8080,
+      path: "/healthz",
+      heartbeat_interval: 5,
+      heartbeat_threshold: 15
+    }.freeze
+
+    def initialize(attrs = {})
+      attrs = DEFAULTS.merge(attrs)
+
+      @enabled = !!attrs[:enabled]
+      @bind = validate_bind(attrs[:bind])
+      @port = validate_port(attrs[:port])
+      @path = validate_path(attrs[:path])
+      @heartbeat_interval = validate_positive_integer(attrs[:heartbeat_interval], :heartbeat_interval)
+      @heartbeat_threshold = validate_threshold(attrs[:heartbeat_threshold], attrs[:heartbeat_interval])
+    end
+
+    def to_h
+      {
+        enabled: enabled,
+        bind: bind,
+        port: port,
+        path: path,
+        heartbeat_interval: heartbeat_interval,
+        heartbeat_threshold: heartbeat_threshold
+      }
+    end
+
+    private
+
+    def validate_bind(value)
+      value = value.to_s
+      raise ArgumentError, "Schked liveness_probe `bind` must be non-empty" if value.empty?
+
+      IPAddr.new(value)
+      value
+    rescue IPAddr::InvalidAddressError
+      raise ArgumentError, "Schked liveness_probe `bind` is invalid: #{value}"
+    end
+
+    def validate_port(value)
+      port = Integer(value)
+      raise ArgumentError, "Schked liveness_probe `port` must be between 1 and 65535, got: #{port}" unless port.between?(1, 65_535)
+
+      port
+    rescue ArgumentError, TypeError
+      raise ArgumentError, "Schked liveness_probe `port` must be an integer between 1 and 65535, got: #{value.inspect}"
+    end
+
+    def validate_path(value)
+      value = value.to_s
+      raise ArgumentError, "Schked liveness_probe `path` must start with /, got: #{value}" unless value.start_with?("/")
+
+      value
+    end
+
+    def validate_positive_integer(value, name)
+      int = Integer(value)
+      raise ArgumentError, "Schked liveness_probe `#{name}` must be a positive integer, got: #{int}" unless int.positive?
+
+      int
+    rescue ArgumentError, TypeError
+      raise ArgumentError, "Schked liveness_probe `#{name}` must be a positive integer, got: #{value.inspect}"
+    end
+
+    def validate_threshold(value, interval)
+      int = validate_positive_integer(value, :heartbeat_threshold)
+      interval = validate_positive_integer(interval, :heartbeat_interval)
+      raise ArgumentError, "Schked liveness_probe `heartbeat_threshold` must be >= heartbeat_interval" if int < interval
+
+      int
+    end
+  end
+
+  class LivenessProbe
+    attr_reader :config, :logger
+
+    def initialize(config:, logger:)
+      @config = config
+      @logger = logger
+      @last_heartbeat_at = nil
+      @shutting_down = false
+      @server = nil
+      @thread = nil
+      @mutex = Mutex.new
+    end
+
+    def start
+      return unless config.enabled
+
+      @server = create_server
+      logger.info("Schked liveness probe listening on #{config.bind}:#{config.port}#{config.path}")
+
+      @thread = Thread.new {
+        Thread.current.name = "schked-liveness-probe" if Thread.current.respond_to?(:name=)
+        accept_loop
+      }
+    end
+
+    def heartbeat
+      @last_heartbeat_at = Time.now
+    end
+
+    def stop
+      @mutex.synchronize do
+        return if @shutting_down
+
+        @shutting_down = true
+      end
+
+      sleep 0.1
+      @server&.close
+      @thread&.join(5)
+      logger.info("Schked liveness probe stopped")
+    end
+
+    def healthy?
+      return false if @shutting_down
+      return false if @last_heartbeat_at.nil?
+
+      Time.now - @last_heartbeat_at <= config.heartbeat_threshold
+    end
+
+    private
+
+    def create_server
+      TCPServer.new(config.bind, config.port)
+    rescue Errno::EADDRINUSE => e
+      raise ArgumentError, "Schked liveness probe port #{config.port} is already in use on #{config.bind}: #{e.message}"
+    end
+
+    def accept_loop
+      loop do
+        client = @server.accept
+
+        Thread.new(client) do |conn|
+          handle_client(conn)
+        end
+      rescue IOError, Errno::EBADF
+        break
+      end
+    end
+
+    def handle_client(client)
+      request_line = read_request_line(client)
+      return if request_line.nil?
+
+      _method, path, _protocol = request_line.split(" ", 3)
+
+      response = response_for(path)
+      client.print(response)
+    ensure
+      begin
+        client.close
+      rescue IOError
+        # already closed
+      end
+    end
+
+    def read_request_line(client)
+      return nil unless IO.select([client], nil, nil, 5)
+
+      client.gets
+    end
+
+    def response_for(path)
+      if path == config.path
+        if healthy?
+          "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK"
+        else
+          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: 11\r\nConnection: close\r\n\r\nUnavailable"
+        end
+      else
+        "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"
+      end
+    end
+  end
+end

data/lib/schked/migration_generator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Schked
+  # Generates the DDL for the +schked_job_runs+ table used by the
+  # database-backed coordination store. Operators copy the SQL into their own
+  # migration; schked does not write migration files or run DDL on its own.
+  module MigrationGenerator
+    module_function
+
+    DDL_POSTGRES = <<~SQL
+      CREATE TABLE schked_job_runs (
+        id BIGSERIAL PRIMARY KEY,
+        job_name TEXT NOT NULL,
+        window_start BIGINT NOT NULL,
+        run_at DOUBLE PRECISION NOT NULL,
+        claimer TEXT NOT NULL,
+        CONSTRAINT schked_job_runs_unique UNIQUE (job_name, window_start)
+      );
+
+      CREATE INDEX schked_job_runs_window_start_idx ON schked_job_runs (window_start);
+    SQL
+
+    DDL_MYSQL = <<~SQL
+      CREATE TABLE schked_job_runs (
+        id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+        job_name VARCHAR(255) NOT NULL,
+        window_start BIGINT NOT NULL,
+        run_at DOUBLE NOT NULL,
+        claimer VARCHAR(255) NOT NULL,
+        UNIQUE KEY schked_job_runs_unique (job_name, window_start),
+        KEY schked_job_runs_window_start_idx (window_start)
+      );
+    SQL
+
+    def sql(flavor = "postgres")
+      case flavor.to_s.downcase
+      when "mysql" then DDL_MYSQL
+      else DDL_POSTGRES
+      end
+    end
+  end
+end

data/lib/schked/railtie.rb

@@ -4,6 +4,8 @@ require "rails/railtie"
 
 module Schked
   class Railtie < Rails::Railtie
+    config.schked = ActiveSupport::OrderedOptions.new
+
     class PathsConfig
       def self.call(app)
         return if Schked.config.do_not_load_root_schedule?
@@ -16,7 +18,15 @@ module Schked
       end
     end
 
+    class LivenessConfig
+      def self.call(app)
+        liveness_probe = app.config.schked.liveness_probe
+        Schked.config.liveness_probe = liveness_probe if liveness_probe
+      end
+    end
+
     initializer("schked.paths", &PathsConfig.method(:call))
+    initializer("schked.liveness", after: :load_config_initializers) { |app| LivenessConfig.call(app) }
 
     config.to_prepare do
       Schked.config.logger = ::Rails.logger unless Schked.config.logger?

data/lib/schked/redis_job_run_store.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Schked
+  # Redis-backed implementation of the per-job coordination store.
+  # Uses +SET key 1 NX EX <ttl>+ so the first caller wins and TTL handles
+  # expiration; #cleanup is a no-op because native TTL covers retention.
+  class RedisJobRunStore
+    include JobRunStore
+
+    KEY_PREFIX = "schked:job_run"
+
+    attr_reader :redis_client, :logger, :max_skew_seconds
+
+    def initialize(redis_client:, logger: Logger.new($stdout), max_skew_seconds: 60)
+      @redis_client = redis_client
+      @logger = logger
+      @max_skew_seconds = Integer(max_skew_seconds)
+    end
+
+    def claim(job_name, window_start)
+      validate!(job_name, window_start)
+
+      key = build_key(job_name, window_start)
+      ttl = default_ttl
+
+      # +SET ... NX EX+ is atomic on a single Redis instance and is the
+      # idiomatic primitive for "claim this slot for at most N seconds".
+      # Transport errors (connection refused, timeout, ...) raise out of
+      # this method so the caller knows the store is unavailable — silently
+      # returning +false+ would skip every job while Redis is down.
+      #
+      # Why not Redlock? Redlock (the algorithm used by +RedisLocker+) is
+      # designed for cluster-wide consensus across multiple Redis masters.
+      # For this dedup, a single +SET NX EX+ is already atomic per
+      # instance and sufficient for exactly-once across the cluster of
+      # *schedulers* — the scheduler cluster itself uses one Redis (or a
+      # single master with replicas).
+      redis_client.call("SET", key, "1", "NX", "EX", ttl) == "OK"
+    end
+
+    def cleanup(_older_than)
+      # Native Redis TTL handles expiration; nothing to do here.
+      nil
+    end
+
+    private
+
+    def default_ttl
+      # The key only needs to outlive the contention window — the interval
+      # (up to +max_skew+, plus scheduling jitter) during which instances
+      # race to claim the same slot. Expiry between windows is harmless:
+      # each window gets its own key. The floor keeps a minimum protective
+      # period for one-shot +at+/+in+ claims against instances that boot
+      # with a delay.
+      [10 * @max_skew_seconds, 3600].max
+    end
+
+    def build_key(job_name, window_start)
+      ts = window_start.is_a?(Time) ? window_start.to_i : Integer(window_start)
+      "#{KEY_PREFIX}:#{job_name}:#{ts}"
+    end
+
+    def validate!(job_name, window_start)
+      raise ArgumentError, "job_name must be a non-empty String" if job_name.to_s.empty?
+      raise ArgumentError, "window_start must not be nil" if window_start.nil?
+    end
+  end
+end

data/lib/schked/schedule_dsl.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "rufus/scheduler"
+
+module Schked
+  # Wraps the rufus-scheduler DSL so deduplication mode (+job_run_store+ is
+  # configured) makes +every+ jobs grid-aligned (via injected +first_at+) and
+  # rejects +interval+ jobs (their phase drifts with job duration and cannot
+  # be deduplicated).
+  #
+  # In the default (non-dedup) mode the wrapper is transparent and forwards
+  # every call to the underlying scheduler.
+  class ScheduleDSL
+    class IntervalNotSupportedError < StandardError; end
+
+    attr_reader :scheduler
+
+    def initialize(scheduler:, dedup_enabled:, max_skew_seconds: 60, logger: Logger.new(File::NULL))
+      @scheduler = scheduler
+      @dedup_enabled = dedup_enabled
+      @max_skew_seconds = Integer(max_skew_seconds)
+      @logger = logger
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      @scheduler.respond_to?(name, include_private) || super
+    end
+
+    def method_missing(name, *args, **kwargs, &block)
+      case name
+      when :every
+        every_with_alignment(*args, **kwargs, &block)
+      when :interval
+        raise IntervalNotSupportedError, interval_error_message if @dedup_enabled
+
+        @scheduler.interval(*args, **kwargs, &block)
+      else
+        @scheduler.public_send(name, *args, **kwargs, &block)
+      end
+    end
+
+    private
+
+    def every_with_alignment(duration, *args, **kwargs, &block)
+      if @dedup_enabled
+        seconds = Rufus::Scheduler.parse_duration(duration)
+        # Shift "now" backward by +max_skew+ so that two instances whose
+        # clocks differ by up to +max_skew+ still land on the same grid
+        # point. Without this shift, an instance that is half a skew
+        # ahead could pick a different slot than one that is half a skew
+        # behind, and both would claim the job.
+        first_at = Time.at(next_grid_point_epoch(seconds, Time.now.to_f - @max_skew_seconds))
+        if kwargs.key?(:first_at)
+          @logger.warn(
+            "Schked: ignoring `first_at: #{kwargs[:first_at].inspect}` for `every` job " \
+            "in deduplication mode — grid alignment is required for claims"
+          )
+        end
+        kwargs = kwargs.merge(first_at: first_at)
+      end
+
+      @scheduler.every(duration, *args, **kwargs, &block)
+    end
+
+    def next_grid_point_epoch(seconds, shifted_now)
+      grid_point = (shifted_now / seconds).ceil * seconds
+      # When the interval is comparable to or smaller than +max_skew+, the
+      # grid point closest to "now − max_skew" may still lie in the past,
+      # and rufus-scheduler rejects a past +first_at+. Advance by whole
+      # periods until the slot is strictly in the future: every step keeps
+      # the value on the absolute (epoch-multiple) grid, so all instances
+      # stay phase-aligned regardless of how many steps they take.
+      grid_point += seconds while grid_point <= Time.now.to_f
+      grid_point
+    end
+
+    def interval_error_message
+      "`interval` jobs are not supported when `Schked.config.job_run_store` is configured. " \
+        "Their phase depends on job duration and cannot be aligned to a stable grid. " \
+        "Use `every` or `cron` instead."
+    end
+  end
+end

data/lib/schked/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Schked
-  VERSION = "1.4.0"
+  VERSION = "2.0.0"
 end

data/lib/schked/worker.rb

@@ -4,17 +4,26 @@ require "rufus/scheduler"
 
 module Schked
   class Worker
+    DEFAULT_CLEANUP_INTERVAL = "60s"
+    DEFAULT_CLEANUP_RETENTION = 24 * 60 * 60 # seconds before cutoff
+
     def initialize(config:)
       @config = config
+      @liveness_probe = nil
 
-      @locker = RedisLocker.new(config.redis, lock_ttl: 40_000, logger: config.logger) unless config.standalone?
+      config.validate!
+      @job_run_store = build_job_run_store
+      @locker = build_locker
 
-      @scheduler = Rufus::Scheduler.new(trigger_lock: locker)
+      scheduler_opts = {trigger_lock: locker}.compact
+      @scheduler = Rufus::Scheduler.new(**scheduler_opts)
 
       watch_signals
-      define_callbacks
-      define_extend_lock unless config.standalone?
+      Callbacks.new(config: config, job_run_store: @job_run_store).install(@scheduler)
+      define_extend_lock if locker
+      define_cleanup_job if database_backed_store?
       load_schedule
+      start_liveness_probe
     end
 
     def job(as)
@@ -30,6 +39,7 @@ module Schked
     end
 
     def stop
+      liveness_probe&.stop
       scheduler.stop
     end
 
@@ -44,41 +54,33 @@ module Schked
 
     private
 
-    attr_reader :config, :scheduler, :locker
-
-    def define_callbacks
-      cfg = config
-
-      scheduler.define_singleton_method(:extract_job_name) do |job|
-        if job
-          job.opts[:as] || job.job_id
-        else
-          "unknown"
-        end
-      end
-
-      scheduler.define_singleton_method(:on_error) do |job, error|
-        cfg.logger.fatal("Task #{extract_job_name(job)} failed with error: #{error.message}")
-        cfg.logger.error(error.backtrace.join("\n")) if error.backtrace
-
-        cfg.fire_callback(:on_error, job, error)
-      end
-
-      scheduler.define_singleton_method(:on_pre_trigger) do |job, time|
-        cfg.logger.info("Started task: #{extract_job_name(job)}")
-
-        cfg.fire_callback(:before_start, job, time)
-      end
-
-      scheduler.define_singleton_method(:around_trigger) do |job, &block|
-        cfg.fire_around_callback(:around_job, job, &block)
+    attr_reader :config, :scheduler, :locker, :liveness_probe, :job_run_store
+
+    def build_job_run_store
+      return nil unless config.dedup_enabled?
+
+      case config.job_run_store
+      when :redis
+        RedisJobRunStore.new(
+          redis_client: RedisClientFactory.build(config.redis),
+          logger: config.logger,
+          max_skew_seconds: config.max_skew
+        )
+      when :database
+        DatabaseConnection.detect(
+          connection: config.database_connection,
+          logger: config.logger
+        )
+      else
+        config.job_run_store
       end
+    end
 
-      scheduler.define_singleton_method(:on_post_trigger) do |job, time|
-        cfg.logger.info("Finished task: #{extract_job_name(job)}")
+    def build_locker
+      return nil if config.standalone?
+      return nil if config.dedup_enabled?
 
-        cfg.fire_callback(:after_finish, job, time)
-      end
+      RedisLocker.new(config.redis, lock_ttl: 40_000, logger: config.logger)
     end
 
     def watch_signals
@@ -94,7 +96,10 @@ module Schked
 
       Thread.new do
         loop do
-          scheduler.shutdown(wait: 5) if @shutdown
+          if @shutdown
+            liveness_probe&.stop
+            scheduler.shutdown(wait: 5)
+          end
           sleep 1
         end
       end
@@ -106,8 +111,49 @@ module Schked
       end
     end
 
+    def define_cleanup_job
+      store = @job_run_store
+      logger = config.logger
+
+      scheduler.every(DEFAULT_CLEANUP_INTERVAL, as: "Schked::Worker#cleanup_job_runs", overlap: false) do
+        cutoff = Time.now.to_i - (DEFAULT_CLEANUP_RETENTION + config.max_skew)
+        logger.debug("Cleaning up database job runs older than #{cutoff}")
+        store.cleanup(cutoff)
+      rescue => e
+        logger.error("Failed to clean up database job runs: #{e.message}")
+      end
+    end
+
+    # Only stores whose +#cleanup+ actually deletes rows need the sweep.
+    # +RedisJobRunStore#cleanup+ is a no-op (native TTL handles retention),
+    # so scheduling it would just log noise on every instance. Custom
+    # stores keep the sweep because they implemented +#cleanup+ for it.
+    # Returns +false+ when no store is configured (default mode).
+    def database_backed_store?
+      !job_run_store.nil? && !job_run_store.is_a?(RedisJobRunStore)
+    end
+
     def load_schedule
-      scheduler.instance_eval(schedule)
+      dsl = ScheduleDSL.new(
+        scheduler: scheduler,
+        dedup_enabled: config.dedup_enabled?,
+        max_skew_seconds: config.max_skew,
+        logger: config.logger
+      )
+      dsl.instance_eval(schedule)
+    end
+
+    def start_liveness_probe
+      return unless config.liveness_probe.enabled
+
+      @liveness_probe = LivenessProbe.new(config: config.liveness_probe, logger: config.logger)
+      @liveness_probe.start
+
+      scheduler.every("#{config.liveness_probe.heartbeat_interval}s", as: "Schked::Worker#liveness_heartbeat", overlap: false) do
+        @liveness_probe.heartbeat
+      end
+
+      @liveness_probe.heartbeat
     end
   end
 end