activejob-temporal 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a0aefb6003d5c49d08c531d80ac7ce5ff82ee038a81aee31d3c5f0d2d438f61
-  data.tar.gz: c0f3f46a2e57f35a0c05290d129d9905f64c6dbe9be30371e79f99d0905d3cd0
+  metadata.gz: 0e1e7c38c2a169353b2d9f7edadca0614e32cb6c8c33ed87fe1d29b72a7dd5cb
+  data.tar.gz: 3255fbe711eac898e2b91a65fa2bc5f858eccda59e605774fa027a6a2ba72291
 SHA512:
-  metadata.gz: 16bd26f8f58993b6c60b21a16f0216fe928eb4ad341aad19483fe77cd3f2c2c21ecf67695969df2137fa2b4953a8f984ff1d446696ce9691e33e0431fa3af177
-  data.tar.gz: b06bd6d8ca772947228dacfe065e5582156d15fccc8501bff3ec14e8f3c18454d22d2b28c09cb1a59788462997067eaa9f12204ac265fd10273468cd653346ef
+  metadata.gz: b722896482be5075f145926ad914fedd43b860d438c0badf855f030aaa501d6839233315e0427b841e24d108e89fe775375f77e8e07f8ac983f43169eac383cd
+  data.tar.gz: 6f40349c1961652d0f4c1e242a383895264cf585a2a6bcf127e3ed6dc66f9025413a664289d3ac05325ed9f10dab0d3f427c240af600555dfab8fb50d37f46fc

data/CHANGELOG.md

@@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - 2026-08-07
+
+### Added
+- `credential_poll_interval` (default 30 seconds) and `credential_file_events` (default false) configure how `tls_cert_watch` and `api_key_watch` detect rotation.
+- `ActiveJob::Temporal::CredentialRefresher` replaces `CertificateWatcher`. Enqueue-side processes can now start it from an initializer instead of scheduling `refresh_api_key!` themselves - see [worker setup](docs/worker_setup.md#api-key-authentication).
+
+### Changed
+- Credential rotation is detected by comparing a content digest on an interval rather than by filesystem events. Rotation on mounts whose events never reach the process (or where the `listen` gem is absent) now works, at up to one poll interval of latency. `credential_file_events = true` restores immediate reaction by waking the check early; it never decides whether a reload happens, so a missing `listen` gem costs latency, not correctness.
+- A cert and key pair rotating together now triggers one client rebuild instead of a debounced pair, and TLS plus API key watching share a single watcher instead of two.
+- **Breaking:** `ActiveJob::Temporal::CertificateWatcher` is removed. Applications constructing it directly should use `CredentialRefresher.from_config`.
+
+### Fixed
+- Watching a Kubernetes projected or secret volume no longer prints listen's `directory is already being watched!` error at boot. kubelet keeps the live secret in a timestamped sibling of the `..data` symlink, so listen's recursive scan reached the same real directory twice; the versioned copy is now skipped.
+
+## [0.3.1] - 2026-08-06
+
+### Fixed
+- `api_key_watch` and `tls_cert_watch` now fire on Kubernetes projected and secret volumes: kubelet rotates them atomically by swapping a `..data` symlink into a new timestamped directory, so the watched file's own path never appears in the change events and the previous exact-path filter dropped every rotation. Any change inside a watched file's directory now counts as relevant.
+
 ## [0.3.0] - 2026-08-06
 
 ### Added

data/activejob-temporal.gemspec

@@ -39,6 +39,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "globalid", ">= 0.3"
   spec.add_dependency "temporalio", ">= 1.4.0", "< 1.7"
 
+  # listen is an optional runtime dependency, required lazily only when credential_file_events is
+  # enabled. Without it credentials still rotate, just on the poll interval instead of instantly.
   spec.add_development_dependency "benchmark-ips", "~> 2.14"
   spec.add_development_dependency "github_changelog_generator", "~> 1.18"
   spec.add_development_dependency "listen", "~> 3.9"

data/bin/temporal-worker

@@ -260,25 +260,14 @@ reload_signal_thread = Thread.new do
 end
 Signal.trap(reload_signal) { reload_signal_queue.push(reload_signal) }
 
-certificate_watcher = nil
-if config.tls_cert_watch
-  certificate_watcher = ActiveJob::Temporal::CertificateWatcher.new(
-    paths: ActiveJob::Temporal::CertificateWatcher.paths_from_config(config),
-    # Failures are logged by the reloader and retried by the watcher, so let them propagate.
-    reload_callback: -> { client_reloader.reload(source: "file_watch") }
-  ).start
-end
-
-# Token rotation mutates the live connection (the SDK sends the header per-RPC from the
-# connection's stored options) - no client rebuild, unlike certificate rotation above.
-# Configuration validation guarantees api_key_file is set when api_key_watch is on.
-api_key_watcher = nil
-if config.api_key_watch
-  api_key_watcher = ActiveJob::Temporal::CertificateWatcher.new(
-    paths: [config.api_key_file],
-    reload_callback: -> { ActiveJob::Temporal.refresh_api_key! }
-  ).start
-end
+# Only the TLS callback is overridden: certificate rotation has to swap the rebuilt client into
+# the running worker, while token rotation mutates the live connection (the SDK sends the header
+# per-RPC from the connection's stored options) and needs no worker awareness.
+credential_refresher = ActiveJob::Temporal::CredentialRefresher.from_config(
+  config,
+  # Failures are logged by the reloader and retried on the next check, so let them propagate.
+  on_tls_change: -> { client_reloader.reload(source: "credential_refresh") }
+).start
 
 begin
   worker_health.mark_started!
@@ -290,8 +279,7 @@ rescue Interrupt
   # Temporal worker raises Interrupt on shutdown signals; swallow to allow clean exit.
 ensure
   worker_health.mark_stopped!
-  certificate_watcher&.stop
-  api_key_watcher&.stop
+  credential_refresher.stop
   reload_signal_queue.close
   reload_signal_thread.join(1)
   health_check_server&.stop

data/lib/activejob/temporal/configuration.rb

@@ -155,8 +155,23 @@ module ActiveJob
         default: false,
         env_var: "ACTIVEJOB_TEMPORAL_API_KEY_WATCH",
         type: :boolean,
-        description: "Watch api_key_file and reload worker clients when it changes (kubelet rewrites " \
-                     "projected tokens in place long before they expire)"
+        description: "Watch api_key_file and refresh the token when it changes (kubelet rotates " \
+                     "projected tokens atomically, long before they expire)"
+      },
+
+      credential_poll_interval: {
+        default: 30,
+        env_var: "ACTIVEJOB_TEMPORAL_CREDENTIAL_POLL_INTERVAL",
+        type: :integer,
+        description: "Seconds between credential file checks when tls_cert_watch or api_key_watch is enabled"
+      },
+
+      credential_file_events: {
+        default: false,
+        env_var: "ACTIVEJOB_TEMPORAL_CREDENTIAL_FILE_EVENTS",
+        type: :boolean,
+        description: "React to credential file changes immediately instead of waiting for the next " \
+                     "poll (requires the optional listen gem; the poll interval stays the safety net)"
       },
 
       priority_task_queues: {
@@ -752,6 +767,7 @@ module ActiveJob
       validate :validate_payload_storage_settings
       validate :validate_tls_settings
       validate :validate_api_key_settings
+      validate :validate_credential_refresh_settings
       validate :validate_worker_registration_settings
       validate :validate_local_activity_helpers
       validate :validate_dependency_wait_settings
@@ -1092,6 +1108,16 @@ module ActiveJob
         errors.add(:api_key_watch, :requires_path)
       end
 
+      def validate_credential_refresh_settings
+        unless [true, false].include?(credential_file_events)
+          errors.add(:credential_file_events, :not_boolean, value: credential_file_events.inspect)
+        end
+
+        return if credential_poll_interval.is_a?(Integer) && credential_poll_interval.positive?
+
+        errors.add(:credential_poll_interval, :invalid, value: credential_poll_interval.inspect)
+      end
+
       def validate_worker_registration_settings
         unless [true, false].include?(worker_activejob_workloads)
           errors.add(:worker_activejob_workloads, :not_boolean, value: worker_activejob_workloads.inspect)

data/lib/activejob/temporal/credential_refresher.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../temporal"
+
+module ActiveJob
+  module Temporal
+    # Keeps credential files fresh in a long-lived process: TLS material, API key files, or
+    # anything else read once at boot and rotated underneath the process later.
+    #
+    # Change detection is a content digest compared on an interval, so the only thing that can fire
+    # a callback is the bytes behind a path actually differing. Filesystem events (see :file_events)
+    # are an optional accelerator that wakes the loop early; they never decide whether a reload
+    # happens, so a missing `listen` gem or a mount without working inotify costs latency, not
+    # correctness.
+    #
+    # @example Worker: rebuild the client on cert rotation, patch the live connection on token rotation
+    #   refresher = CredentialRefresher.from_config(
+    #     ActiveJob::Temporal.config,
+    #     on_tls_change: -> { client_reloader.reload(source: "credential_refresh") }
+    #   ).start
+    #
+    # @example Enqueue-side process (web, Sidekiq) with a rotating projected token
+    #   require "activejob/temporal/credential_refresher"
+    #   ActiveJob::Temporal::CredentialRefresher.from_config(ActiveJob::Temporal.config).start
+    class CredentialRefresher
+      # kubelet keeps the live secret in a timestamped sibling of `..data`, so listen's recursive
+      # scan reaches the same real directory twice and prints a SymlinkLoop error to stderr.
+      # Getting this wrong costs that stderr noise and nothing else: listen only ever nudges the
+      # loop, so the digest still decides, and the interval still backs it up.
+      #
+      # @api private
+      KUBERNETES_VERSIONED_DIR = /\A\.\.\d/
+      private_constant :KUBERNETES_VERSIONED_DIR
+
+      DEFAULT_POLL_INTERVAL = 30
+
+      # Files digested as one unit. Cert and key rotate as separate writes, so grouping them turns
+      # a rotation into a single callback instead of a pair.
+      Source = Struct.new(:name, :paths, :on_change, keyword_init: true)
+
+      class << self
+        # Builds a refresher from the `tls_cert_watch` / `api_key_watch` configuration flags.
+        # The callbacks default to the process-wide reload paths, which is what an enqueue-side
+        # process wants; workers override the TLS one so the running worker gets the new client.
+        #
+        # @param configuration [Configuration] the gem configuration
+        # @param on_tls_change [#call] invoked when any TLS file changes
+        # @param on_api_key_change [#call] invoked when the API key file changes
+        # @param logger [#log_event, #warn, #error] structured logger
+        # @param listener_factory [#to, nil] injection point for tests, defaults to `Listen`
+        # @return [CredentialRefresher] a refresher with no sources when both flags are off
+        def from_config(configuration,
+                        on_tls_change: -> { ActiveJob::Temporal.reload_client! },
+                        on_api_key_change: -> { ActiveJob::Temporal.refresh_api_key! },
+                        logger: ActiveJob::Temporal::Logger,
+                        listener_factory: nil)
+          sources = []
+
+          if configuration.tls_cert_watch
+            sources << Source.new(name: "tls", paths: tls_paths(configuration), on_change: on_tls_change)
+          end
+
+          if configuration.api_key_watch
+            sources << Source.new(name: "api_key", paths: [configuration.api_key_file], on_change: on_api_key_change)
+          end
+
+          new(
+            sources: sources,
+            poll_interval: configuration.credential_poll_interval,
+            file_events: configuration.credential_file_events,
+            logger: logger,
+            listener_factory: listener_factory
+          )
+        end
+
+        # @param configuration [Configuration] the gem configuration
+        # @return [Array<String>] configured TLS file paths, blanks removed
+        def tls_paths(configuration)
+          [
+            configuration.tls_cert_path,
+            configuration.tls_key_path,
+            configuration.tls_server_root_ca_cert_path
+          ].compact.reject { |path| path.to_s.strip.empty? }
+        end
+      end
+
+      # @param sources [Array<Source>] credential groups to keep fresh
+      # @param poll_interval [Integer, Float] seconds between digest comparisons
+      # @param file_events [Boolean] wake the loop early on filesystem events, requires `listen`
+      # @param logger [#log_event, #warn, #error] structured logger
+      # @param listener_factory [#to, nil] injection point for tests, defaults to `Listen`
+      def initialize(sources:, poll_interval: DEFAULT_POLL_INTERVAL, file_events: false,
+                     logger: ActiveJob::Temporal::Logger, listener_factory: nil)
+        @sources = sources.reject { |source| source.paths.compact.empty? }
+        @poll_interval = poll_interval
+        @file_events = file_events
+        @logger = logger
+        @listener_factory = listener_factory
+        @wakeups = Thread::Queue.new
+        @digests = {}
+        @thread = nil
+        @listener = nil
+      end
+
+      # Baselines each digest before polling, so a process booting on an already-rotated file does
+      # not fire a redundant reload on its first check.
+      #
+      # @return [self]
+      def start
+        return self if @sources.empty? || @thread
+
+        @sources.each { |source| @digests[source.name] = digest(source) }
+        @thread = Thread.new { run }
+        start_listener if @file_events
+        self
+      end
+
+      # @return [void]
+      def stop
+        @listener&.stop
+        @listener = nil
+        @wakeups.close
+        @thread&.join
+        @thread = nil
+      end
+
+      # Re-reads every source and fires the callbacks whose content changed. Safe to call at any
+      # rate: unchanged content is a no-op.
+      #
+      # @return [void]
+      def refresh_changed_sources
+        @sources.each { |source| refresh(source) }
+      end
+
+      # Wakes the poll loop so the next comparison happens now instead of on the next tick.
+      # Decides nothing on its own.
+      #
+      # @return [void]
+      def nudge
+        @wakeups << :wakeup
+      rescue ClosedQueueError
+        nil
+      end
+
+      private
+
+      def run
+        until @wakeups.closed?
+          @wakeups.pop(timeout: @poll_interval)
+          break if @wakeups.closed?
+
+          # A rotation emits a burst of events; collapse them into the single check that follows.
+          @wakeups.clear
+          refresh_changed_sources
+        end
+      end
+
+      def refresh(source)
+        current = digest(source)
+        return if current.nil? || current == @digests[source.name]
+
+        source.on_change.call
+        # Stamped only after the callback succeeds, so a failed reload is retried on the next tick.
+        @digests[source.name] = current
+        @logger.log_event("credential_refreshed", source: source.name)
+      rescue StandardError => e
+        @logger.error(
+          "credential_refresh_failed",
+          source: source.name,
+          error_class: e.class.name,
+          message: e.message
+        )
+      end
+
+      # A path can be briefly unreadable while kubelet swaps the mount. Treat that as "no reading"
+      # so the next tick retries, rather than reloading against a half-written credential.
+      def digest(source)
+        contents = source.paths.compact.map { |path| TLSFile.read(path) }
+        Digest::SHA256.hexdigest(contents.join("\0"))
+      rescue TLSFile::Error
+        nil
+      end
+
+      def start_listener
+        @listener = listener_factory.to(*watched_directories, ignore: KUBERNETES_VERSIONED_DIR) { nudge }
+        @listener.start
+      rescue LoadError => e
+        # Polling already guarantees the refresh, so a missing listen gem is a latency
+        # regression rather than a boot failure.
+        @logger.warn(
+          "credential_file_events_unavailable",
+          error_class: e.class.name,
+          message: e.message,
+          poll_interval_seconds: @poll_interval
+        )
+      end
+
+      def watched_directories
+        @sources.flat_map(&:paths).compact.map { |path| File.dirname(File.expand_path(path)) }.uniq
+      end
+
+      def listener_factory
+        @listener_factory ||= begin
+          require "listen"
+          Listen
+        end
+      end
+    end
+  end
+end

data/lib/activejob/temporal/locales/en.yml

@@ -145,6 +145,12 @@ en:
               not_boolean: "must be true or false, got: %{value}"
               requires_path: "requires api_key_file to watch"
 
+            credential_poll_interval:
+              invalid: "must be a positive number of seconds, got: %{value}"
+
+            credential_file_events:
+              not_boolean: "must be true or false, got: %{value}"
+
             worker_activejob_workloads:
               not_boolean: "must be true or false, got: %{value}"
               requires_activities: "requires worker_activities when disabled - the worker would have nothing to register"

data/lib/activejob/temporal/version.rb

@@ -2,6 +2,6 @@
 
 module ActiveJob
   module Temporal
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/activejob/temporal/worker_runtime.rb

@@ -2,7 +2,7 @@
 
 require_relative "../temporal"
 require_relative "rails_environment_loader"
-require_relative "certificate_watcher"
+require_relative "credential_refresher"
 require_relative "reload_signal_queue"
 require_relative "worker_client_reloader"
 require_relative "worker_pool"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activejob-temporal
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Temporal Technologies
@@ -309,7 +309,6 @@ files:
 - lib/activejob/temporal/cancel.rb
 - lib/activejob/temporal/cancel/batch_canceller.rb
 - lib/activejob/temporal/cancel/batch_summary.rb
-- lib/activejob/temporal/certificate_watcher.rb
 - lib/activejob/temporal/chain_options.rb
 - lib/activejob/temporal/child_workflow_options.rb
 - lib/activejob/temporal/client.rb
@@ -318,6 +317,7 @@ files:
 - lib/activejob/temporal/configuration.rb
 - lib/activejob/temporal/configured_job_compatibility.rb
 - lib/activejob/temporal/connection_worker_pool.rb
+- lib/activejob/temporal/credential_refresher.rb
 - lib/activejob/temporal/dead_letter_payload_validation.rb
 - lib/activejob/temporal/dead_letter_queue.rb
 - lib/activejob/temporal/dependency_options.rb

data/lib/activejob/temporal/certificate_watcher.rb

@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-module ActiveJob
-  module Temporal
-    # Watches TLS certificate files and runs a reload callback when they change.
-    class CertificateWatcher
-      DEFAULT_DEBOUNCE_SECONDS = 1.0
-
-      def self.paths_from_config(configuration)
-        [
-          configuration.tls_cert_path,
-          configuration.tls_key_path,
-          configuration.tls_server_root_ca_cert_path
-        ].compact.reject { |path| path.to_s.strip.empty? }
-      end
-
-      def initialize(paths:, reload_callback:, listener_factory: nil, debounce_seconds: DEFAULT_DEBOUNCE_SECONDS)
-        @paths = paths.map { |path| File.expand_path(path) }.uniq
-        @reload_callback = reload_callback
-        @listener_factory = listener_factory
-        @debounce_seconds = debounce_seconds
-        @mutex = Mutex.new
-        @last_reload_at = nil
-        @listener = nil
-        @trailing_reload = nil
-      end
-
-      def start
-        return self if @paths.empty? || @listener
-
-        @listener = listener_factory.to(*directories) do |modified, added, removed|
-          handle_changes(modified + added + removed)
-        end
-        @listener.start
-        self
-      end
-
-      def stop
-        @listener&.stop
-        @listener = nil
-        @mutex.synchronize { @trailing_reload }&.kill
-      end
-
-      def handle_changes(changed_paths)
-        return unless relevant_change?(changed_paths)
-
-        if debounced?
-          schedule_trailing_reload
-        else
-          reload(retry_on_failure: true)
-        end
-      end
-
-      private
-
-      # A failed reload clears the debounce stamp so the next change reloads
-      # immediately, and gets one retry in case no further change arrives.
-      def reload(retry_on_failure:)
-        @reload_callback.call
-      rescue StandardError
-        @mutex.synchronize { @last_reload_at = nil }
-        schedule_trailing_reload if retry_on_failure
-      end
-
-      # Cert and key rotate as separate writes, so the second one lands inside
-      # the debounce window. Flush it once the window closes instead of dropping it.
-      def schedule_trailing_reload
-        @mutex.synchronize do
-          return if @trailing_reload&.alive?
-
-          @trailing_reload = Thread.new do
-            sleep(@debounce_seconds)
-            @mutex.synchronize { @last_reload_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) }
-            reload(retry_on_failure: false)
-          end
-        end
-      end
-
-      def directories
-        @directories ||= @paths.map { |path| File.dirname(path) }.uniq
-      end
-
-      def listener_factory
-        @listener_factory ||= begin
-          require "listen"
-          Listen
-        rescue LoadError => e
-          raise LoadError, "listen gem is required when tls_cert_watch is enabled: #{e.message}"
-        end
-      end
-
-      def relevant_change?(changed_paths)
-        changed_paths.any? do |path|
-          @paths.include?(File.expand_path(path))
-        end
-      end
-
-      def debounced?
-        return false unless @debounce_seconds.positive?
-
-        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        @mutex.synchronize do
-          return true if @last_reload_at && (now - @last_reload_at) < @debounce_seconds
-
-          @last_reload_at = now
-          false
-        end
-      end
-    end
-  end
-end