ci-queue 0.99.0 → 0.101.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12bdc1e58974c1c3fca314133800774ca993f23fd01440931443d080cc23ee6f
-  data.tar.gz: 6e4f28118f8281ff3959f631e2e8b7c6fc8fbc47d0b1692f533e866edbfce916
+  metadata.gz: 612d0f2f3077fdd5b7e74b343649a162101c2961c8b4cda8329ca10fbd831e6b
+  data.tar.gz: bdd01c95971bcbfab5999b3754f8d019be35a7202e5f766d5df19191462e5563
 SHA512:
-  metadata.gz: 88f858df9c99323f322479bd0f20f41d28137d45ee216a45f5f49317128bbfbe5312f5becee71ef559e03e0ea377ddf8e59e1e7ac177aa735b3c2b024e48f7e3
-  data.tar.gz: e8738af5e5031e7fbe2c5118ea9c08c9b9e5943078d8b0ead60d6b72ca9c2cb29a3dfe847f03d83b8824fd7cd2e6b6b323a35e05904710bf9774626ebfdfb5f5
+  metadata.gz: 69a03983e12372fe0ab0258ec08262b2672624d49d2d63a0c35d0a89b4c8c85b3fd89ece2ce3c71de112652cd327868ee0fa7633ae283facb3b0bf57035d5724
+  data.tar.gz: 44ead0bb47b690b19652982496f8a48d698f5bb78ea4f88c01e56b7774d4c79f6b6015b50e91cb70e241fc00a13bf842a72828792c7f716644cacf12bfb8bb2d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.99.0)
+    ci-queue (0.101.0)
       logger
 
 GEM

data/README.md

@@ -260,3 +260,17 @@ After merging changes to `main`, follow these steps to release and propagate the
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
 
 You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)
+
+## Redis over TLS
+
+Use a `rediss://` queue URL to connect to Redis over TLS. The server certificate and hostname are verified using the system's trusted CAs.
+
+> **Upgrading from 0.99.0 or earlier:** those versions did not verify certificates for `rediss://` URLs. If your Redis presents a self-signed or otherwise untrusted certificate, connections now fail with `certificate verify failed`. Workers treat this like an unreachable Redis: they report `Ran 0 tests` and exit successfully, and the error only surfaces when `report` fails with `Redis::CannotConnectError`. Trust the issuing CA or opt out as described below.
+
+If your Redis server presents a certificate signed by a private CA, point OpenSSL at that CA (for example with `SSL_CERT_FILE=/path/to/ca.pem`).
+
+If your hosted Redis only offers a self-signed certificate that can't be verified, you can opt out of verification with `CI_QUEUE_REDIS_SSL_VERIFY=0` (or `CI::Queue::Configuration#redis_ssl_verify = false`). The connection stays encrypted, but the server isn't authenticated, so anyone who can intercept traffic between your CI workers and Redis can impersonate the server and read the credentials in the queue URL. Only use this when you accept that risk.
+
+| Environment variable | Description |
+|---|---|
+| `CI_QUEUE_REDIS_SSL_VERIFY=0` | Disable TLS certificate verification for `rediss://` connections. Defaults to enabled. No CLI equivalent. |

data/ci-queue.gemspec

@@ -3,10 +3,11 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'ci/queue/version'
-require 'pathname'
 
-dir = Pathname.new(CI::Queue::RELEASE_SCRIPTS_ROOT).relative_path_from(Pathname.new(__dir__).realpath)
-lua_scripts = Dir[dir.join('*.lua').to_s]
+# Release Lua scripts are gitignored copies made by `rake scripts:copy`, so
+# `git ls-files` misses them. Glob without touching CI::Queue constants:
+# Dependabot evaluates this file with the `require` above rescued away.
+lua_scripts = Dir.glob('lib/ci/queue/redis/*.lua', base: __dir__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'ci-queue'

data/lib/ci/queue/configuration.rb

@@ -11,6 +11,7 @@ module CI
       attr_writer :lazy_load_streaming_timeout
       attr_accessor :lazy_load_test_helpers
       attr_accessor :skip_stale_tests
+      attr_accessor :redis_ssl_verify
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -34,6 +35,7 @@ module CI
             lazy_load_streaming_timeout: (env['CI_QUEUE_LAZY_LOAD_STREAM_TIMEOUT'] || env['CI_QUEUE_STREAM_TIMEOUT'])&.to_i,
             lazy_load_test_helpers: env['CI_QUEUE_LAZY_LOAD_TEST_HELPERS'] || env['CI_QUEUE_TEST_HELPERS'],
             skip_stale_tests: %w(1 true).include?(env['CI_QUEUE_SKIP_STALE_TESTS']&.strip&.downcase),
+            redis_ssl_verify: !%w(0 false).include?(env['CI_QUEUE_REDIS_SSL_VERIFY']&.strip&.downcase),
           )
         end
 
@@ -60,7 +62,7 @@ module CI
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
         export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
         lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
-        skip_stale_tests: false)
+        skip_stale_tests: false, redis_ssl_verify: true)
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
         @failure_file = failure_file
@@ -94,6 +96,7 @@ module CI
         @lazy_load_streaming_timeout = lazy_load_streaming_timeout
         @lazy_load_test_helpers = lazy_load_test_helpers
         @skip_stale_tests = skip_stale_tests
+        @redis_ssl_verify = redis_ssl_verify
       end
 
       def lazy_load_test_helper_paths

data/lib/ci/queue/redis/base.rb

@@ -41,19 +41,33 @@ module CI
               # it makes sense to retry for a while before giving up.
               reconnect_attempts: reconnect_attempts,
               middlewares: custom_middlewares,
-              # Hosted Redis servers use self signed certificates
-              # (because they do not own the domain they're running on such as compute-1.amazonaws.com)
-              # therefore a full SSL connection verification will fail.
-              # ci-queue should not contain any sensitive data, so we can just disable the verification.
-              ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE },
+              ssl_params: self.class.redis_ssl_params(ssl_verify: redis_ssl_verify?),
               custom: custom_config,
               timeout: DEFAULT_TIMEOUT,
             )
           else
-            @redis = ::Redis.new(url: redis_url, timeout: DEFAULT_TIMEOUT)
+            @redis = ::Redis.new(
+              url: redis_url,
+              timeout: DEFAULT_TIMEOUT,
+              ssl_params: self.class.redis_ssl_params(ssl_verify: redis_ssl_verify?),
+            )
           end
         end
 
+        # TLS (`rediss://`) connections verify the server certificate and hostname
+        # by default. Some hosted Redis providers present self-signed certificates;
+        # those deployments can either trust the issuing CA (e.g. via `SSL_CERT_FILE`)
+        # or explicitly opt out of verification with `CI_QUEUE_REDIS_SSL_VERIFY=0`.
+        def self.redis_ssl_params(ssl_verify:)
+          ssl_verify ? {} : { verify_mode: OpenSSL::SSL::VERIFY_NONE }
+        end
+
+        def redis_ssl_verify?
+          return true unless @config.respond_to?(:redis_ssl_verify)
+
+          @config.redis_ssl_verify != false
+        end
+
         def reconnect_attempts
           return [] if ENV["CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS"]
 
@@ -271,8 +285,9 @@ module CI
           # on every heartbeat, which fires once per running test per worker.
           TICK_COMMAND = 'tick!'.freeze
 
-          def initialize(redis_url, zset_key, owners_key, leases_key)
+          def initialize(redis_url, zset_key, owners_key, leases_key, ssl_verify: true)
             @redis_url = redis_url
+            @ssl_verify = ssl_verify
             @zset_key = zset_key
             @owners_key = owners_key
             @leases_key = leases_key
@@ -283,6 +298,7 @@ module CI
             ready_pipe, child_write = IO.pipe
             @pipe.binmode
             @pid = Process.spawn(
+              { 'CI_QUEUE_REDIS_SSL_VERIFY' => @ssl_verify ? '1' : '0' },
               RbConfig.ruby,
               ::File.join(__dir__, "monitor.rb"),
               @redis_url,
@@ -380,6 +396,7 @@ module CI
             key('running'),
             key('owners'),
             key('leases'),
+            ssl_verify: redis_ssl_verify?,
           )
         end
 

data/lib/ci/queue/redis/monitor.rb

@@ -3,6 +3,7 @@
 # frozen_string_literal: true
 
 require 'logger'
+require 'openssl'
 require 'redis'
 require 'json'
 
@@ -18,7 +19,11 @@ module CI
           @owners_key = owners_key
           @leases_key = leases_key
           @logger = logger
-          @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
+          @redis = ::Redis.new(
+            url: redis_url,
+            reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+            ssl_params: ssl_params,
+          )
           @shutdown = false
           @pipe = pipe
           @self_pipe_reader, @self_pipe_writer = IO.pipe
@@ -30,6 +35,17 @@ module CI
           end
         end
 
+        # Mirrors CI::Queue::Redis::Base.redis_ssl_params. This script runs as a
+        # standalone subprocess that doesn't load ci-queue or its Configuration,
+        # so the parent passes the setting through the environment.
+        def ssl_params
+          if %w(0 false).include?(ENV['CI_QUEUE_REDIS_SSL_VERIFY']&.strip&.downcase)
+            { verify_mode: OpenSSL::SSL::VERIFY_NONE }
+          else
+            {}
+          end
+        end
+
         def soft_signal(sig)
           @queue << sig
           @self_pipe_writer << '.'

data/lib/ci/queue/redis/requeue.lua

@@ -12,10 +12,30 @@ local leases_key = KEYS[9]
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local entry = ARGV[3]
-local offset = ARGV[4]
+local offset = tonumber(ARGV[4]) or 0
 local ttl = tonumber(ARGV[5])
 local lease_id = ARGV[6]
 
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with reserve.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
+  end
+end
+
 -- Only the current lease holder can requeue a test.
 -- If the lease was transferred (e.g. via reserve_lost), reject the stale
 -- worker's requeue so the running entry stays intact for the new holder.
@@ -42,12 +62,7 @@ redis.call('hincrby', requeues_count_key, entry, 1)
 
 redis.call('hdel', error_reports_key, entry)
 
-local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
-if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
-else
-  redis.call('lpush', queue_key, entry)
-end
+insert_with_offset(queue_key, entry, offset)
 
 redis.call('hset', requeued_by_key, entry, worker_queue_key)
 if ttl and ttl > 0 then

data/lib/ci/queue/redis/reserve.lua

@@ -13,12 +13,23 @@ local current_time = ARGV[1]
 local defer_offset = tonumber(ARGV[2]) or 0
 local max_skip_attempts = 4
 
-local function insert_with_offset(test)
-  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
-  if pivot then
-    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
-  else
-    redis.call('lpush', queue_key, test)
+-- Inserts `entry` behind the `offset` + 1 entries nearest the tail, the end RPOP reserves
+-- from: where `LINSERT BEFORE <entry at index -(offset + 1)>` would put it. LINSERT finds its
+-- pivot by scanning from the head, which is O(queue length) and blocks Redis on large queues;
+-- this only touches the tail, so it is O(offset). Queues of at most `offset` + 1 entries, and
+-- offsets of zero or less, push to the head instead.
+-- Keep in sync with requeue.lua: the Python client does not resolve `-- @include`.
+local function insert_with_offset(queue_key, entry, offset)
+  if offset <= 0 or redis.call('llen', queue_key) <= offset + 1 then
+    redis.call('lpush', queue_key, entry)
+    return
+  end
+
+  local ahead = redis.call('lrange', queue_key, -1 - offset, -1)
+  redis.call('ltrim', queue_key, 0, -2 - offset)
+  redis.call('rpush', queue_key, entry)
+  for _, ahead_entry in ipairs(ahead) do
+    redis.call('rpush', queue_key, ahead_entry)
   end
 end
 
@@ -45,7 +56,7 @@ for attempt = 1, max_skip_attempts do
       return claim_test(test)
     end
 
-    insert_with_offset(test)
+    insert_with_offset(queue_key, test, defer_offset)
 
     -- If this worker only finds its own requeued tests, defer once by returning nil,
     -- then allow pickup on a subsequent reserve attempt.

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.99.0'
+    VERSION = '0.101.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.99.0
+  version: 0.101.0
 platform: ruby
 authors:
 - Jean Boussier