ci-queue 0.99.0 → 0.100.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12bdc1e58974c1c3fca314133800774ca993f23fd01440931443d080cc23ee6f
-  data.tar.gz: 6e4f28118f8281ff3959f631e2e8b7c6fc8fbc47d0b1692f533e866edbfce916
+  metadata.gz: 64510328261d26c0b3299aa0e1bb4bbbb2fdd4f784293a304cec028527d25e8b
+  data.tar.gz: 8cf5e99fcf74935b71a2ee6814cfc7565b03b1b314af8380f999f4228400417c
 SHA512:
-  metadata.gz: 88f858df9c99323f322479bd0f20f41d28137d45ee216a45f5f49317128bbfbe5312f5becee71ef559e03e0ea377ddf8e59e1e7ac177aa735b3c2b024e48f7e3
-  data.tar.gz: e8738af5e5031e7fbe2c5118ea9c08c9b9e5943078d8b0ead60d6b72ca9c2cb29a3dfe847f03d83b8824fd7cd2e6b6b323a35e05904710bf9774626ebfdfb5f5
+  metadata.gz: c044a9cafe05602f22debf2ea91b5cc63c42dabc67f8689b88430c7c34ece68da7bf7ad03b62036648073c8225e3588382bce588f486359b339a65aa62e71951
+  data.tar.gz: 5d9f475df1260d6cc6cd04d1f1ceec0ca28e7b12e9d5d0b0931ac7b18a2e864a5a2386f102f878a8392722dbb535e260b8661e5ccc5722097ee8d8e866d977dc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.99.0)
+    ci-queue (0.100.0)
       logger
 
 GEM

data/README.md

@@ -260,3 +260,17 @@ After merging changes to `main`, follow these steps to release and propagate the
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
 
 You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)
+
+## Redis over TLS
+
+Use a `rediss://` queue URL to connect to Redis over TLS. The server certificate and hostname are verified using the system's trusted CAs.
+
+> **Upgrading from 0.99.0 or earlier:** those versions did not verify certificates for `rediss://` URLs. If your Redis presents a self-signed or otherwise untrusted certificate, connections now fail with `certificate verify failed`. Workers treat this like an unreachable Redis: they report `Ran 0 tests` and exit successfully, and the error only surfaces when `report` fails with `Redis::CannotConnectError`. Trust the issuing CA or opt out as described below.
+
+If your Redis server presents a certificate signed by a private CA, point OpenSSL at that CA (for example with `SSL_CERT_FILE=/path/to/ca.pem`).
+
+If your hosted Redis only offers a self-signed certificate that can't be verified, you can opt out of verification with `CI_QUEUE_REDIS_SSL_VERIFY=0` (or `CI::Queue::Configuration#redis_ssl_verify = false`). The connection stays encrypted, but the server isn't authenticated, so anyone who can intercept traffic between your CI workers and Redis can impersonate the server and read the credentials in the queue URL. Only use this when you accept that risk.
+
+| Environment variable | Description |
+|---|---|
+| `CI_QUEUE_REDIS_SSL_VERIFY=0` | Disable TLS certificate verification for `rediss://` connections. Defaults to enabled. No CLI equivalent. |

data/ci-queue.gemspec

@@ -3,10 +3,11 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'ci/queue/version'
-require 'pathname'
 
-dir = Pathname.new(CI::Queue::RELEASE_SCRIPTS_ROOT).relative_path_from(Pathname.new(__dir__).realpath)
-lua_scripts = Dir[dir.join('*.lua').to_s]
+# Release Lua scripts are gitignored copies made by `rake scripts:copy`, so
+# `git ls-files` misses them. Glob without touching CI::Queue constants:
+# Dependabot evaluates this file with the `require` above rescued away.
+lua_scripts = Dir.glob('lib/ci/queue/redis/*.lua', base: __dir__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'ci-queue'

data/lib/ci/queue/configuration.rb

@@ -11,6 +11,7 @@ module CI
       attr_writer :lazy_load_streaming_timeout
       attr_accessor :lazy_load_test_helpers
       attr_accessor :skip_stale_tests
+      attr_accessor :redis_ssl_verify
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -34,6 +35,7 @@ module CI
             lazy_load_streaming_timeout: (env['CI_QUEUE_LAZY_LOAD_STREAM_TIMEOUT'] || env['CI_QUEUE_STREAM_TIMEOUT'])&.to_i,
             lazy_load_test_helpers: env['CI_QUEUE_LAZY_LOAD_TEST_HELPERS'] || env['CI_QUEUE_TEST_HELPERS'],
             skip_stale_tests: %w(1 true).include?(env['CI_QUEUE_SKIP_STALE_TESTS']&.strip&.downcase),
+            redis_ssl_verify: !%w(0 false).include?(env['CI_QUEUE_REDIS_SSL_VERIFY']&.strip&.downcase),
           )
         end
 
@@ -60,7 +62,7 @@ module CI
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
         export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
         lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
-        skip_stale_tests: false)
+        skip_stale_tests: false, redis_ssl_verify: true)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -94,6 +96,7 @@ module CI
         @lazy_load_streaming_timeout = lazy_load_streaming_timeout
         @lazy_load_test_helpers = lazy_load_test_helpers
         @skip_stale_tests = skip_stale_tests
+        @redis_ssl_verify = redis_ssl_verify
       end
 
       def lazy_load_test_helper_paths

data/lib/ci/queue/redis/base.rb

@@ -41,19 +41,33 @@ module CI
               # it makes sense to retry for a while before giving up.
               reconnect_attempts: reconnect_attempts,
               middlewares: custom_middlewares,
-              # Hosted Redis servers use self signed certificates
-              # (because they do not own the domain they're running on such as compute-1.amazonaws.com)
-              # therefore a full SSL connection verification will fail.
-              # ci-queue should not contain any sensitive data, so we can just disable the verification.
-              ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE },
+              ssl_params: self.class.redis_ssl_params(ssl_verify: redis_ssl_verify?),
               custom: custom_config,
               timeout: DEFAULT_TIMEOUT,
             )
           else
-            @redis = ::Redis.new(url: redis_url, timeout: DEFAULT_TIMEOUT)
+            @redis = ::Redis.new(
+              url: redis_url,
+              timeout: DEFAULT_TIMEOUT,
+              ssl_params: self.class.redis_ssl_params(ssl_verify: redis_ssl_verify?),
+            )
           end
         end
 
+        # TLS (`rediss://`) connections verify the server certificate and hostname
+        # by default. Some hosted Redis providers present self-signed certificates;
+        # those deployments can either trust the issuing CA (e.g. via `SSL_CERT_FILE`)
+        # or explicitly opt out of verification with `CI_QUEUE_REDIS_SSL_VERIFY=0`.
+        def self.redis_ssl_params(ssl_verify:)
+          ssl_verify ? {} : { verify_mode: OpenSSL::SSL::VERIFY_NONE }
+        end
+
+        def redis_ssl_verify?
+          return true unless @config.respond_to?(:redis_ssl_verify)
+
+          @config.redis_ssl_verify != false
+        end
+
         def reconnect_attempts
           return [] if ENV["CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS"]
 
@@ -271,8 +285,9 @@ module CI
           # on every heartbeat, which fires once per running test per worker.
           TICK_COMMAND = 'tick!'.freeze
 
-          def initialize(redis_url, zset_key, owners_key, leases_key)
+          def initialize(redis_url, zset_key, owners_key, leases_key, ssl_verify: true)
             @redis_url = redis_url
+            @ssl_verify = ssl_verify
             @zset_key = zset_key
             @owners_key = owners_key
             @leases_key = leases_key
@@ -283,6 +298,7 @@ module CI
             ready_pipe, child_write = IO.pipe
             @pipe.binmode
             @pid = Process.spawn(
+              { 'CI_QUEUE_REDIS_SSL_VERIFY' => @ssl_verify ? '1' : '0' },
               RbConfig.ruby,
               ::File.join(__dir__, "monitor.rb"),
               @redis_url,
@@ -380,6 +396,7 @@ module CI
             key('running'),
             key('owners'),
             key('leases'),
+            ssl_verify: redis_ssl_verify?,
           )
         end
 

data/lib/ci/queue/redis/monitor.rb

@@ -3,6 +3,7 @@
 # frozen_string_literal: true
 
 require 'logger'
+require 'openssl'
 require 'redis'
 require 'json'
 
@@ -18,7 +19,11 @@ module CI
           @owners_key = owners_key
           @leases_key = leases_key
           @logger = logger
-          @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
+          @redis = ::Redis.new(
+            url: redis_url,
+            reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+            ssl_params: ssl_params,
+          )
           @shutdown = false
           @pipe = pipe
           @self_pipe_reader, @self_pipe_writer = IO.pipe
@@ -30,6 +35,17 @@ module CI
           end
         end
 
+        # Mirrors CI::Queue::Redis::Base.redis_ssl_params. This script runs as a
+        # standalone subprocess that doesn't load ci-queue or its Configuration,
+        # so the parent passes the setting through the environment.
+        def ssl_params
+          if %w(0 false).include?(ENV['CI_QUEUE_REDIS_SSL_VERIFY']&.strip&.downcase)
+            { verify_mode: OpenSSL::SSL::VERIFY_NONE }
+          else
+            {}
+          end
+        end
+
         def soft_signal(sig)
           @queue << sig
           @self_pipe_writer << '.'

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.99.0'
+    VERSION = '0.100.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.99.0
+  version: 0.100.0
 platform: ruby
 authors:
 - Jean Boussier