gitlab-experiment 1.4.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1bda42d269c22b469ab0a942c38e01b3d718db2c4e7c4ffa61a41e0da53e74a
-  data.tar.gz: 020d283e9875365e5db338e1dfdfe57e3cf53e10a46d6f2d069603931f94bf50
+  metadata.gz: 0e0e710a80403ae390bb66fd552656996ca271891cbc4b2cd758f064538442e5
+  data.tar.gz: 564ea70f574ebabef344f4f428cfad52a6213a35522834956db8eb8073a498e9
 SHA512:
-  metadata.gz: 558f12b6f92de538d024757f271194675d0ed863e50325debdcf16ddb2e20873df92e008dc20818962a5bbe536aba97c923345119c4a55f2855fc428e0cd43fa
-  data.tar.gz: dcebe030c7d8b232c5b4c261c894f9db21b48744c80b807642d07a772b9cc0e26ee94264e1a3aa2680326f94f7ae06f860349d0521449d3f2145974c6b3a84d9
+  metadata.gz: b76011392395d1d0b0e1d7beeec6dc085ea6d7444c231b9dbbdac7f909248cce4ca16f551bb11215b2611ad504a63a593669b31e544d69d1658da0e0ee6004eb
+  data.tar.gz: e9c96959bc9c30b2684bb4eb13665032753f97463f11192b66f2c5592bbe66c29a49663d1fa4a11b2dc64493d10b462dc24b6a6b2380899467df68cdac26ccae

data/README.md

@@ -680,11 +680,17 @@ end
 ```
 
 **Behavior with `only_assigned: true`:**
-- ✅ If user already assigned → returns their cached variant
-- ✅ If user not assigned → returns control, no tracking
+- ✅ If user has a cached assignment (control or candidate) → returns that variant and tracks
+- ✅ If user has no cached assignment → returns control, no tracking
 - ✅ Experiment reach stays controlled
 - ✅ Perfect for multi-page experimental experiences
 
+> **Note:** `only_assigned: true` relies on cached assignments. Control variants are cached
+> by default (see [Advanced: Caching Configuration](#advanced-caching-configuration)) so that
+> bucketed control users are included in `only_assigned` tracking. If you set
+> `cache_control: false`, control users will appear unassigned at secondary sites and be
+> excluded from tracking there.
+
 **Real-world use cases:**
 - **Post-signup experiences**: Assign at signup, show features throughout the app
 - **Gradual feature expansion**: Roll out to 5%, then add more touchpoints without expanding population
@@ -1318,6 +1324,34 @@ documented in its implementation.
 
 **Important:** Caching changes how rollout strategies behave. Once cached, subsequent calls return the cached value regardless of rollout strategy changes.
 
+**Control variant caching:**
+
+By default, the gem caches `:control` assignments alongside variant assignments. This is what
+makes `only_assigned: true` lookups include bucketed control users -- without it, control
+users would appear unassigned at secondary tracking sites and would be silently excluded.
+
+```ruby
+# Disable globally (don't cache :control)
+Gitlab::Experiment.configure do |config|
+  config.cache_control = false
+end
+
+# Or override per-rollout
+class MyExperiment < ApplicationExperiment
+  default_rollout :percent, cache_control: false
+end
+```
+
+The `Random` rollout strategy intentionally opts out of caching `:control` regardless of this
+setting, because its convergence model depends on re-rolling control assignments on every
+visit until a non-control variant is picked.
+
+**Operational note:** With control caching enabled, `Percent` and `RoundRobin` cache hashes
+grow with every bucketed user, not just users assigned to a candidate variant. For
+long-running experiments the cache will be roughly `1 / candidate_percentage` the size it
+would have been before. Plan periodic cleanup using your store's deletion API -- for the
+`RedisHashStore`, that's `experiment.cache.store.clear(key: experiment.name)`.
+
 ### Advanced: Custom Rollout Strategies
 
 Build custom integrations with your existing infrastructure:

data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt

@@ -22,6 +22,20 @@ Gitlab::Experiment.configure do |config|
   # Use `nil` for no caching.
   config.cache = nil
 
+  # Cache `:control` assignments alongside variant assignments.
+  #
+  # When set to true (the default), bucketed control users are cached and
+  # therefore visible to `only_assigned: true` tracking calls. Set to false
+  # to skip caching control -- control users will be treated as unassigned
+  # at `only_assigned` sites and excluded from tracking there.
+  #
+  # Can be overridden per-rollout:
+  #
+  # class ExampleExperiment < ApplicationExperiment
+  #   default_rollout :percent, cache_control: false
+  # end
+  config.cache_control = true
+
   # The domain to use on cookies.
   #
   # When not set, it uses the current host. If you want to provide specific

data/lib/gitlab/experiment/base_interface.rb

@@ -81,7 +81,7 @@ module Gitlab
       def process_redirect_url(url)
         return unless Configuration.redirect_url_validator&.call(url)
 
-        track('visited', url: url)
+        track('visited', url:)
         url # return the url, which allows for mutation
       end
 

data/lib/gitlab/experiment/cache.rb

@@ -43,18 +43,18 @@ module Gitlab
         @cache ||= Interface.new(self, Configuration.cache)
       end
 
-      def cache_variant(specified = nil, &block)
+      def cache_variant(specified = nil, &)
         return (specified.presence || yield) unless cache.store
 
-        result = migrated_cache_fetch(cache.store) || find_variant(&block)
+        result = migrated_cache_fetch(cache.store) || find_variant(&)
         return result unless specified.present?
 
         cache.write(specified) if result.to_s != specified.to_s
         specified
       end
 
-      def find_variant(&block)
-        cache.store.fetch(cache_key, &block)
+      def find_variant(&)
+        cache.store.fetch(cache_key, &)
       end
 
       def cache_key(key = nil, suffix: nil)

data/lib/gitlab/experiment/configuration.rb

@@ -30,6 +30,21 @@ module Gitlab
       # Use `nil` for no caching.
       @cache = nil
 
+      # Whether `:control` assignments should be written to the cache.
+      #
+      # When true (default), control variants are cached alongside other
+      # variants, which is required for `only_assigned: true` tracking to
+      # work correctly for the control arm of an experiment.
+      #
+      # When false, control variants are not cached. This preserves the
+      # behavior of versions prior to this option being introduced. Any
+      # experiment using `only_assigned: true` will silently exclude
+      # control users from tracking.
+      #
+      # Individual rollouts can override this via the `cache_control:`
+      # option, e.g. `default_rollout :percent, cache_control: false`.
+      @cache_control = true
+
       # The domain to use on cookies.
       #
       # When not set, it uses the current host. If you want to provide specific
@@ -126,7 +141,7 @@ module Gitlab
       # access experiment methods, like `name`, `context`, and `signature`.
       @tracking_behavior = lambda do |event, args|
         # An example of using a generic logger to track events:
-        Configuration.logger.info("#{self.class.name}[#{name}] #{event}: #{args.merge(signature: signature)}")
+        Configuration.logger.info("#{self.class.name}[#{name}] #{event}: #{args.merge(signature:)}")
 
         # Using something like snowplow to track events (in gitlab):
         #
@@ -158,7 +173,7 @@ module Gitlab
       #   level1 initiated by file_name.rb:2
       #   level2 initiated by file_name.rb:3
       @nested_behavior = lambda do |nested_experiment|
-        raise NestingError.new(experiment: self, nested_experiment: nested_experiment)
+        raise NestingError.new(experiment: self, nested_experiment:)
       end
 
       # Called at the end of every experiment run, with the result.
@@ -193,6 +208,7 @@ module Gitlab
           :base_class,
           :strict_registration,
           :cache,
+          :cache_control,
           :cookie_domain,
           :cookie_name,
           :secure_cookie,
@@ -213,6 +229,19 @@ module Gitlab
           @default_rollout = Rollout.resolve(*args)
         end
 
+        def cache_control=(value) # rubocop:disable Lint/DuplicateMethods
+          if value != true
+            deprecated(
+              :cache_control,
+              "setting `cache_control` to a non-true value is deprecated and will be removed in version 2.0. " \
+                "Control variants will always be cached.",
+              version: '1.4'
+            )
+          end
+
+          @cache_control = value
+        end
+
         # Internal warning helpers.
 
         def deprecated(*args, version:, stack: 0)

data/lib/gitlab/experiment/context.rb

@@ -49,7 +49,7 @@ module Gitlab
       end
 
       def signature
-        @signature ||= { key: key, migration_keys: migration_keys }.compact
+        @signature ||= { key:, migration_keys: }.compact
       end
 
       def method_missing(method_name, *)

data/lib/gitlab/experiment/cookies.rb

@@ -22,7 +22,7 @@ module Gitlab
         return hash.merge(key => cookie) if hash[key].nil?
 
         add_unmerged_migration(key => cookie)
-        cookie_jar.delete(cookie_name, domain: domain)
+        cookie_jar.delete(cookie_name, domain:)
 
         hash
       end
@@ -32,7 +32,7 @@ module Gitlab
 
         cookie ||= SecureRandom.uuid
         cookie_jar.permanent.signed[cookie_name] = {
-          value: cookie, secure: Configuration.secure_cookie, domain: domain, httponly: true
+          value: cookie, secure: Configuration.secure_cookie, domain:, httponly: true
         }
 
         hash.merge(key => cookie)

data/lib/gitlab/experiment/errors.rb

@@ -7,6 +7,7 @@ module Gitlab
     UnregisteredExperiment = Class.new(Error)
     ExistingBehaviorError = Class.new(Error)
     BehaviorMissingError = Class.new(Error)
+    ControlNotCachedError = Class.new(Error)
 
     class NestingError < Error
       def initialize(experiment:, nested_experiment:)

data/lib/gitlab/experiment/rollout/random.rb

@@ -3,9 +3,12 @@
 # The random rollout strategy will randomly assign a variant when the context is determined to be within the experiment
 # group.
 #
-# If caching is enabled this is a predicable and consistent assignment that will eventually assign a variant (since
-# control isn't cached) but if caching isn't enabled, assignment will be random each time.
+# If caching is enabled this is a predictable and consistent assignment that will eventually assign a non-control
+# variant. This relies on the strategy intentionally NOT caching control assignments -- so when a context is sampled
+# into control, it gets re-rolled on the next visit until it lands on a non-control variant, at which point the
+# assignment is cached and stable. Without caching, assignment is random on every call.
 #
+# This is the only rollout that opts out of caching control: every other rollout caches whatever variant it resolves to.
 # Example configuration usage:
 #
 # config.default_rollout = Gitlab::Experiment::Rollout::Random.new
@@ -28,7 +31,10 @@ module Gitlab
         protected
 
         def execute_assignment
-          behavior_names.sample # pick a random variant
+          result = behavior_names.sample # pick a random variant
+          # Drop :control so it's never cached, even if cache_control is true.
+          # The context will be re-rolled on the next visit and eventually converge on a non-control variant.
+          result == :control ? nil : result
         end
       end
     end

data/lib/gitlab/experiment/rollout.rb

@@ -31,6 +31,15 @@ module Gitlab
 
           @experiment = experiment
           @options = options
+
+          return if !options.key?(:cache_control) || options[:cache_control] == true
+
+          Configuration.deprecated(
+            :cache_control,
+            "setting `cache_control` to a non-true value on a rollout is deprecated and " \
+              "will be removed in version 2.0. Control variants will always be cached.",
+            version: '1.4'
+          )
         end
 
         def enabled?
@@ -41,7 +50,17 @@ module Gitlab
           validate! # allow the rollout strategy to validate itself
 
           assignment = execute_assignment
-          assignment == :control ? nil : assignment # avoid caching control by returning nil
+          # Cache control assignments unless explicitly opted out. The per-rollout `cache_control:`
+          # option takes precedence over the global `Configuration.cache_control` default.
+          return assignment if cache_control_enabled?
+
+          assignment == :control ? nil : assignment
+        end
+
+        # Returns true when this rollout caches `:control` assignments. Per-rollout
+        # `cache_control:` option takes precedence over the global default.
+        def cache_control_enabled?
+          options.fetch(:cache_control, Configuration.cache_control)
         end
 
         private

data/lib/gitlab/experiment/rspec.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'active_support/core_ext/enumerable'
+require 'active_support/core_ext/object/blank'
+
 module Gitlab
   class Experiment
     module TestBehaviors
@@ -24,7 +27,7 @@ module Gitlab
       end
 
       module MethodDouble
-        def proxy_method_invoked(receiver, *args, &block)
+        def proxy_method_invoked(receiver, *args, &)
           RSpecMocks.track_gitlab_experiment_receiver(original_method, receiver)
           super
         end
@@ -121,6 +124,38 @@ module Gitlab
       end
     end
 
+    # Records experiment tracking calls captured via Configuration.tracking_behavior, so the
+    # `have_tracked_experiment` matcher can assert on the ordered sequence of emitted events. Only
+    # GLEX-native data is captured (the action plus the experiment's signature), so it works for any
+    # consumer regardless of how the host application wires up tracking.
+    class TrackedExperiments
+      attr_reader :events
+
+      def initialize(max_segmentations = 1)
+        @events = []
+        @max_segmentations = max_segmentations
+      end
+
+      def record(action, signature)
+        event = { action: }.merge(signature)
+        verify_single_assignment(event)
+        @events << event
+      end
+
+      private
+
+      def verify_single_assignment(event)
+        return unless event[:action] == :assignment && event[:key].present?
+
+        keys = @events.select { |e| e[:action] == :assignment && e[:key].present? }
+          .pluck(:key).push(event[:key]).uniq
+        return unless keys.size > @max_segmentations
+
+        raise "#{event[:experiment]} was segmented #{keys.size} " \
+          "#{keys.size == 1 ? 'time' : 'times'} (expected #{@max_segmentations}) - #{keys.join(', ')}."
+      end
+    end
+
     module RSpecMatchers
       extend RSpec::Matchers::DSL
 
@@ -135,6 +170,10 @@ module Gitlab
         experiment
       end
 
+      def tracked_experiments
+        @_tracked_experiments
+      end
+
       matcher :register_behavior do |behavior_name|
         match do |experiment|
           @experiment = require_experiment(experiment, 'register_behavior')
@@ -262,7 +301,7 @@ module Gitlab
         chain(:on_next_instance) { @on_next_instance = true }
 
         def set_expectations(event, *event_args, negated:)
-          failure_message = failure_message_with_details(event, negated: negated)
+          failure_message = failure_message_with_details(event, negated:)
           expectations = proc do |e|
             allow(e).to receive(:track).and_call_original
 
@@ -306,6 +345,43 @@ module Gitlab
           details.unshift(base).join("\n")
         end
       end
+
+      matcher :have_tracked_experiment do |experiment_name, expectation = []|
+        match do
+          @experiment_name = experiment_name
+          @expectation = expectation
+
+          expectation == formatted_events
+        end
+
+        failure_message do
+          "Expected to track ordered events:\n#{events_to_s(@expectation)}\n" \
+            "but tracked:\n#{events_to_s(formatted_events)}"
+        end
+
+        def formatted_events
+          unless tracked_experiments
+            raise 'have_tracked_experiment requires the :experiment_tracking metadata tag on the example'
+          end
+
+          tracked_experiments.events
+            .select { |event| event[:experiment] == @experiment_name.to_s }
+            .uniq { |event| event[:action] }
+            .map.with_index { |event, index| format_event(event, @expectation[index] || {}) }
+        end
+
+        # A bare expectation entry (eg. `:assignment`) asserts only the action; a Hash entry (eg.
+        # `{ action: :assignment, migration_keys: [...] }`) asserts each named signature field.
+        def format_event(event, expected_event)
+          return event[:action] unless expected_event.is_a?(Hash)
+
+          expected_event.keys.index_with { |key| event[key] }
+        end
+
+        def events_to_s(events)
+          events.map { |event| "   #{event}" }.join(",\n")
+        end
+      end
     end
   end
 end
@@ -326,6 +402,26 @@ RSpec.configure do |config|
 
   config.include Gitlab::Experiment::RSpecMatchers, :experiment
   config.include Gitlab::Experiment::RSpecMatchers, type: :experiment
+  config.include Gitlab::Experiment::RSpecMatchers, :experiment_tracking
+
+  # Capture experiment tracking calls at the gem's own emission seam (Configuration.tracking_behavior),
+  # so `have_tracked_experiment` can assert on them without coupling to any host tracking implementation.
+  # The metadata value, when numeric, sets the maximum number of segmentations allowed (defaults to 1).
+  config.before(:each, :experiment_tracking) do |example|
+    max_segmentations = example.metadata[:experiment_tracking]
+    max_segmentations = 1 unless max_segmentations.is_a?(Numeric)
+    recorder = @_tracked_experiments = Gitlab::Experiment::TrackedExperiments.new(max_segmentations)
+
+    @_original_tracking_behavior = Gitlab::Experiment::Configuration.tracking_behavior
+    Gitlab::Experiment::Configuration.tracking_behavior = lambda do |action, _args|
+      # `instance_exec`'d on the experiment by `Experiment#track`, so `signature` is in scope here.
+      recorder.record(action, signature)
+    end
+  end
+
+  config.after(:each, :experiment_tracking) do
+    Gitlab::Experiment::Configuration.tracking_behavior = @_original_tracking_behavior
+  end
 
   config.define_derived_metadata(file_path: Regexp.new('spec/experiments/')) do |metadata|
     metadata[:type] ||= :experiment

data/lib/gitlab/experiment/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   class Experiment
-    VERSION = '1.4.0'
+    VERSION = '1.6.0'
   end
 end

data/lib/gitlab/experiment.rb

@@ -33,16 +33,16 @@ module Gitlab
     class << self
       # Class level behavior registration methods.
 
-      def control(*filter_list, **options, &block)
-        variant(:control, *filter_list, **options, &block)
+      def control(*filter_list, **options, &)
+        variant(:control, *filter_list, **options, &)
       end
 
-      def candidate(*filter_list, **options, &block)
-        variant(:candidate, *filter_list, **options, &block)
+      def candidate(*filter_list, **options, &)
+        variant(:candidate, *filter_list, **options, &)
       end
 
-      def variant(variant, *filter_list, **options, &block)
-        build_behavior_callback(filter_list, variant, **options, &block)
+      def variant(variant, *filter_list, **options, &)
+        build_behavior_callback(filter_list, variant, **options, &)
       end
 
       # Class level callback registration methods.
@@ -86,12 +86,12 @@ module Gitlab
       [Configuration.name_prefix, @_name].compact.join('_')
     end
 
-    def control(&block)
-      variant(:control, &block)
+    def control(&)
+      variant(:control, &)
     end
 
-    def candidate(&block)
-      variant(:candidate, &block)
+    def candidate(&)
+      variant(:candidate, &)
     end
 
     def variant(name, &block)
@@ -166,7 +166,16 @@ module Gitlab
     end
 
     def only_assigned?
-      !!context.only_assigned && find_variant.blank?
+      return false unless context.only_assigned
+
+      unless rollout.cache_control_enabled?
+        raise ControlNotCachedError,
+          "`only_assigned: true` requires `cache_control: true`, but the rollout for " \
+            "experiment `#{name}` has it disabled. Control users would be silently excluded " \
+            "from tracking. Enable cache_control or remove `only_assigned: true`."
+      end
+
+      find_variant.blank?
     end
 
     def should_track?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-experiment
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.0
 platform: ruby
 authors:
 - GitLab
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-05-22 00:00:00.000000000 Z
+date: 2026-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -246,7 +246,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="