gitlab-experiment 0.6.4 → 0.7.1

data/lib/generators/gitlab/experiment/experiment_generator.rb

@@ -8,10 +8,15 @@ module Gitlab
       source_root File.expand_path('templates/', __dir__)
       check_class_collision suffix: 'Experiment'
 
-      argument :variants,
-               type: :array,
-               default: %w[control candidate],
-               banner: 'variant variant'
+      argument     :variants,
+                   type: :array,
+                   default: %w[control candidate],
+                   banner: 'variant variant'
+
+      class_option :skip_comments,
+                   type: :boolean,
+                   default: false,
+                   desc: 'Omit helpful comments from generated files'
 
       def create_experiment
         template 'experiment.rb', File.join('app/experiments', class_path, "#{file_name}_experiment.rb")

data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt

@@ -1,18 +1,25 @@
 # frozen_string_literal: true
 
 Gitlab::Experiment.configure do |config|
-  # Prefix all experiment names with a given value. Use `nil` for none.
+  # Prefix all experiment names with a given string value.
+  # Use `nil` for no prefix.
   config.name_prefix = nil
 
-  # The logger is used to log various details of the experiments.
+  # The logger can be used to log various details of the experiments.
   config.logger = Logger.new($stdout)
 
-  # The base class that should be instantiated for basic experiments. It should
-  # be a string, so we can constantize it later.
+  # The base class that should be instantiated for basic experiments.
+  # It should be a string, so we can constantize it later.
   config.base_class = 'ApplicationExperiment'
 
-  # The caching layer is expected to respond to fetch, like Rails.cache for
-  # instance -- or anything that adheres to ActiveSupport::Cache::Store.
+  # Require experiments to be defined in a class, with variants registered.
+  # This will disallow any anonymous experiments that are run inline
+  # without previously defining a class.
+  config.strict_registration = false
+
+  # The caching layer is expected to match the Rails.cache interface.
+  # If no cache is provided some rollout strategies may behave differently.
+  # Use `nil` for no caching.
   config.cache = nil
 
   # The domain to use on cookies.
@@ -24,62 +31,71 @@ Gitlab::Experiment.configure do |config|
   #   nil, :all, or ['www.gitlab.com', '.gitlab.com']
   config.cookie_domain = :all
 
-  # The default rollout strategy that works for single and multi-variants.
+  # The default rollout strategy.
+  #
+  # The recommended default rollout strategy when not using caching would
+  # be `Gitlab::Experiment::Rollout::Percent` as that will consistently
+  # assign the same variant with or without caching.
+  #
+  # Gitlab::Experiment::Rollout::Base can be inherited to implement your
+  # own rollout strategies.
   #
-  # You can provide your own rollout strategies and override them per
-  # experiment.
+  # Each experiment can specify its own rollout strategy:
   #
-  # Examples include:
-  #   Rollout::Random, or Rollout::RoundRobin
-  config.default_rollout = Gitlab::Experiment::Rollout::Percent
+  # class ExampleExperiment < ApplicationExperiment
+  #   default_rollout :random,              # :percent, :round_robin,
+  #                   include_control: true # or MyCustomRollout
+  # end
+  #
+  # Included rollout strategies:
+  #   :percent (recommended), :round_robin, or :random
+  config.default_rollout = :percent, {
+    include_control: true # include control in possible assignments
+  )
 
   # Secret seed used in generating context keys.
   #
+  # You'll typically want to use an environment variable or secret value
+  # for this.
+  #
   # Consider not using one that's shared with other systems, like Rails'
-  # SECRET_KEY_BASE. Generate a new secret and utilize that instead.
-  @context_key_secret = nil
+  # SECRET_KEY_BASE for instance. Generate a new secret and utilize that
+  # instead.
+  config.context_key_secret = nil
 
   # Bit length used by SHA2 in generating context keys.
   #
   # Using a higher bit length would require more computation time.
   #
   # Valid bit lengths:
-  #   256, 384, or 512.
-  @context_key_bit_length = 256
+  #   256, 384, or 512
+  config.context_key_bit_length = 256
 
   # The default base path that the middleware (or rails engine) will be
-  # mounted. Can be nil if you don't want anything to be mounted automatically.
+  # mounted. The middleware enables an instrumentation url, that's similar
+  # to links that can be instrumented in email campaigns.
   #
-  # This enables a similar behavior to how links are instrumented in emails.
+  # Use `nil` if you don't want to mount the middleware.
   #
   # Examples:
   #   '/-/experiment', '/redirect', nil
   config.mount_at = '/experiment'
 
   # When using the middleware, links can be instrumented and redirected
-  # elsewhere. This can be exploited to make a harmful url look innocuous or
-  # that it's a valid url on your domain. To avoid this, you can provide your
-  # own logic for what urls will be considered valid and redirected to.
+  # elsewhere. This can be exploited to make a harmful url look innocuous
+  # or that it's a valid url on your domain. To avoid this, you can provide
+  # your own logic for what urls will be considered valid and redirected
+  # to.
   #
   # Expected to return a boolean value.
-  config.redirect_url_validator = lambda do |redirect_url|
+  config.redirect_url_validator = lambda do |_redirect_url|
     true
   end
 
-  # Logic this project uses to determine inclusion in a given experiment.
-  #
-  # Expected to return a boolean value.
-  #
-  # This block is executed within the scope of the experiment and so can access
-  # experiment methods, like `name`, `context`, and `signature`.
-  config.inclusion_resolver = lambda do |requested_variant|
-    false
-  end
-
   # Tracking behavior can be implemented to link an event to an experiment.
   #
-  # This block is executed within the scope of the experiment and so can access
-  # experiment methods, like `name`, `context`, and `signature`.
+  # This block is executed within the scope of the experiment and so can
+  # access experiment methods, like `name`, `context`, and `signature`.
   config.tracking_behavior = lambda do |event, args|
     # An example of using a generic logger to track events:
     config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
@@ -93,26 +109,52 @@ Gitlab::Experiment.configure do |config|
     # ))
   end
 
+  # Logic designed to respond when a given experiment is nested within
+  # another experiment. This can be useful to identify overlaps and when a
+  # code path leads to an experiment being nested within another.
+  #
+  # Reporting complexity can arise when one experiment changes rollout, and
+  # a downstream experiment is impacted by that.
+  #
+  # The base_class or a custom experiment can provide a `nest_experiment`
+  # method that implements its own logic that may allow certain experiments
+  # to be nested within it.
+  #
+  # This block is executed within the scope of the experiment and so can
+  # access experiment methods, like `name`, `context`, and `signature`.
+  #
+  # The default exception will include the where the experiment calls were
+  # initiated on, so for instance:
+  #
+  # Gitlab::Experiment::NestingError: unable to nest level2 within level1:
+  #   level1 initiated by file_name.rb:2
+  #   level2 initiated by file_name.rb:3
+  config.nested_behavior = lambda do |nested_experiment|
+    raise Gitlab::Experiment::NestingError.new(experiment: self, nested_experiment: nested_experiment)
+  end
+
   # Called at the end of every experiment run, with the result.
   #
-  # You may want to track that you've assigned a variant to a given context,
-  # or push the experiment into the client or publish results elsewhere like
-  # into redis.
+  # You may want to track that you've assigned a variant to a given
+  # context, or push the experiment into the client or publish results
+  # elsewhere like into redis.
   #
-  # This block is executed within the scope of the experiment and so can access
-  # experiment methods, like `name`, `context`, and `signature`.
+  # This block is executed within the scope of the experiment and so can
+  # access experiment methods, like `name`, `context`, and `signature`.
   config.publishing_behavior = lambda do |result|
     # Track the event using our own configured tracking logic.
     track(:assignment)
 
-    # Push the experiment knowledge into the front end. The signature contains
-    # the context key, and the variant that has been determined.
+    # Log using our logging system, so the result (which can be large) can
+    # be reviewed later if we want to.
     #
-    # Gon.push({ experiment: { name => signature } }, true)
+    # Lograge::Event.log(experiment: name, result: result, signature: signature)
 
-    # Log using our logging system, so the result (which can be large) can be
-    # reviewed later if we want to.
+    # Experiments that have been run during the request lifecycle can be
+    # pushed to the client layer by injecting the published experiments
+    # into javascript in a layout or view using something like:
     #
-    # Lograge::Event.log(experiment: name, result: result, signature: signature)
+    # = javascript_tag(nonce: content_security_policy_nonce) do
+    #   window.experiments = #{raw Gitlab::Experiment.published_experiments.to_json};
   end
 end

data/lib/generators/gitlab/experiment/templates/experiment.rb.tt

@@ -6,10 +6,76 @@ require_dependency "<%= namespaced_path %>/application_experiment"
 <% end -%>
 <% module_namespacing do -%>
 class <%= class_name %>Experiment < ApplicationExperiment
+  # Describe your experiment:
+  #
+  # The variant behaviors defined here will be called whenever the experiment
+  # is run unless overrides are provided.
+
 <% variants.each do |variant| -%>
-  def <%= variant %>_behavior
-  end
-<%= "\n" unless variant == variants.last -%>
+<% if %w[control candidate].include?(variant) -%>
+  <%= variant %> { }
+<% else -%>
+  variant(:<%= variant %>) { }
+<% end -%>
+<% end -%>
+
+<% unless options[:skip_comments] -%>
+  # You can register a `control`, `candidate`, or by naming variants directly.
+  # All of these can be registered using blocks, or by specifying a method.
+  #
+  # Here's some ways you might want to register your control logic:
+  #
+  #control { 'class level control' } # yield this block
+  #control :custom_control # call a private method
+  #control # call the private `control_behavior` method
+  #
+  # You can register candidate logic in the same way:
+  #
+  #candidate { 'class level candidate' } # yield this block
+  #candidate :custom_candidate # call a private method
+  #candidate # call the private `candidate_behavior` method
+  #
+  # For named variants it's the same, but a variant name must be provided:
+  #
+  #variant(:example) { 'class level example variant' }
+  #variant(:example) :example_variant
+  #variant(:example) # call the private `example_behavior` method
+  #
+  # Advanced customization:
+  #
+  # Some additional tools are provided to exclude and segment contexts. To
+  # exclude a given context, you can provide rules. For example, we could
+  # exclude all old accounts and all users with a specific first name.
+  #
+  #exclude :old_account?, ->{ context.user.first_name == 'Richard' }
+  #
+  # Segmentation allows for logic to be used to determine which variant a
+  # context will be assigned. Let's say you want to put all old accounts into a
+  # specific variant, and all users with a specific first name in another:
+  #
+  #segment :old_account?, variant: :variant_two
+  #segment(variant: :variant_one) { context.actor.first_name == 'Richard' }
+  #
+  # Utilizing your experiment:
+  #
+  # Once you've defined your experiment, you can run it elsewhere. You'll want
+  # to specify a context (you can read more about context here), and overrides
+  # for any or all of the variants you've registered in your experiment above.
+  #
+  # Here's an example of running the experiment that's sticky to current_user,
+  # with an override for our class level candidate logic:
+  #
+  #   experiment(:<%= file_name %>, user: current_user) do |e|
+  #     e.candidate { 'override <%= class_name %>Experiment behavior' }
+  #   end
+  #
+  # If you want to publish the experiment to the client without running any
+  # code paths on the server, you can simply call publish instead of passing an
+  # experimental block:
+  #
+  #   experiment(:<%= file_name %>, project: project).publish
+  #
+
 <% end -%>
 end
 <% end -%>

data/lib/gitlab/experiment/base_interface.rb

@@ -4,7 +4,6 @@ module Gitlab
   class Experiment
     module BaseInterface
       extend ActiveSupport::Concern
-      include Scientist::Experiment
 
       class_methods do
         def configure
@@ -24,7 +23,19 @@ module Gitlab
         def constantize(name = nil)
           return self if name.nil?
 
-          experiment_name(name).classify.safe_constantize || Configuration.base_class.constantize
+          experiment_class = experiment_name(name).classify
+          experiment_class.safe_constantize || begin
+            return Configuration.base_class.constantize unless Configuration.strict_registration
+
+            raise UnregisteredExperiment, <<~ERR
+              No experiment registered for `#{name}`. Please register the experiment by defining a class:
+
+              class #{experiment_class} < #{Configuration.base_class}
+                control
+                candidate { 'candidate' }
+              end
+            ERR
+          end
         end
 
         def from_param(id)
@@ -37,59 +48,110 @@ module Gitlab
       def initialize(name = nil, variant_name = nil, **context)
         raise ArgumentError, 'name is required' if name.blank? && self.class.base?
 
-        @name = self.class.experiment_name(name, suffix: false)
-        @context = Context.new(self, **context)
-        @variant_name = cache_variant(variant_name) { nil } if variant_name.present?
-
-        compare { false }
+        @_name = self.class.experiment_name(name, suffix: false)
+        @_context = Context.new(self, **context)
+        @_assigned_variant_name = cache_variant(variant_name) { nil } if variant_name.present?
 
         yield self if block_given?
       end
 
       def inspect
-        "#<#{self.class.name || 'AnonymousClass'}:#{format('0x%016X', __id__)} @name=#{name} @context=#{context.value}>"
+        "#<#{self.class.name || 'AnonymousClass'}:#{format('0x%016X', __id__)} name=#{name} context=#{context.value}>"
+      end
+
+      def run(variant_name)
+        behaviors.freeze
+        context.freeze
+
+        block = behaviors[variant_name]
+        raise BehaviorMissingError, "the `#{variant_name}` variant hasn't been registered" if block.nil?
+
+        result = block.call
+        publish(result) if enabled?
+
+        result
       end
 
       def id
         "#{name}:#{context.key}"
       end
-      alias_method :session_id, :id
-      alias_method :to_param, :id
 
-      def flipper_id
-        "Experiment;#{id}"
-      end
+      alias_method :to_param, :id
 
       def variant_names
-        @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
+        @_variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
       end
 
       def behaviors
-        @behaviors ||= public_methods.each_with_object(super) do |name, behaviors|
+        @_behaviors ||= public_behaviors_with_deprecations(registered_behavior_callbacks)
+      end
+
+      # @deprecated
+      def public_behaviors_with_deprecations(behaviors)
+        named_variants = %w[control candidate]
+        public_methods.each_with_object(behaviors) do |name, behaviors|
+          name = name.to_s # fixes compatibility for ruby 2.6.x
           next unless name.end_with?('_behavior')
 
-          behavior_name = name.to_s.sub(/_behavior$/, '')
+          behavior_name = name.sub(/_behavior$/, '')
+          registration = named_variants.include?(behavior_name) ? behavior_name : "variant :#{behavior_name}"
+
+          Configuration.deprecated(<<~MESSAGE, version: '0.7.0', stack: 2)
+            using a public `#{name}` method is deprecated and will be removed from {{release}}, instead register variants using:
+
+            class #{self.class.name} < #{Configuration.base_class}
+              #{registration}
+
+              private
+
+              def #{name}
+                #...
+              end
+            end
+          MESSAGE
+
           behaviors[behavior_name] ||= -> { send(name) } # rubocop:disable GitlabSecurity/PublicSend
         end
       end
 
-      protected
+      # @deprecated
+      def session_id
+        Configuration.deprecated(:session_id, 'instead use `id` or use a custom rollout strategy', version: '0.7.0')
+        id
+      end
 
-      def raise_on_mismatches?
-        false
+      # @deprecated
+      def flipper_id
+        Configuration.deprecated(:flipper_id, 'instead use `id` or use a custom rollout strategy', version: '0.7.0')
+        "Experiment;#{id}"
       end
 
+      # @deprecated
+      def use(&block)
+        Configuration.deprecated(:use, 'instead use `control`', version: '0.7.0')
+
+        control(&block)
+      end
+
+      # @deprecated
+      def try(name = nil, &block)
+        if name.present?
+          Configuration.deprecated(:try, "instead use `variant(:#{name})`", version: '0.7.0')
+          variant(name, &block)
+        else
+          Configuration.deprecated(:try, 'instead use `candidate`', version: '0.7.0')
+          candidate(&block)
+        end
+      end
+
+      protected
+
       def cached_variant_resolver(provided_variant)
         return :control if excluded?
 
         result = cache_variant(provided_variant) { resolve_variant_name }
         result.to_sym if result.present?
       end
-
-      def generate_result(variant_name)
-        observation = Scientist::Observation.new(variant_name, self, &behaviors[variant_name])
-        Scientist::Result.new(self, [observation], observation)
-      end
     end
   end
 end

data/lib/gitlab/experiment/cache/redis_hash_store.rb

@@ -1,25 +1,25 @@
 # frozen_string_literal: true
 
-require 'active_support/notifications'
-
-# This cache strategy is an implementation on top of the redis hash data type,
-# that also adheres to the ActiveSupport::Cache::Store interface. It's a good
-# example of how to build a custom caching strategy for Gitlab::Experiment, and
-# is intended to be a long lived cache -- until the experiment is cleaned up.
+# This cache strategy is an implementation on top of the redis hash data type, that also adheres to the
+# ActiveSupport::Cache::Store interface. It's a good example of how to build a custom caching strategy for
+# Gitlab::Experiment, and is intended to be a long lived cache -- until the experiment is cleaned up.
 #
 # The data structure:
 #   key: experiment.name
 #   fields: context key => variant name
 #
-# Gitlab::Experiment::Configuration.cache = Gitlab::Experiment::Cache::RedisHashStore.new(
-#   pool: -> { Gitlab::Redis::SharedState.with { |redis| yield redis } }
+# Example configuration usage:
+#
+# config.cache = Gitlab::Experiment::Cache::RedisHashStore.new(
+#   pool: ->(&block) { block.call(Redis.current) }
 # )
+#
 module Gitlab
   class Experiment
     module Cache
       class RedisHashStore < ActiveSupport::Cache::Store
-        # Clears the entire cache for a given experiment. Be careful with this
-        # since it would reset all resolved variants for the entire experiment.
+        # Clears the entire cache for a given experiment. Be careful with this since it would reset all resolved
+        # variants for the entire experiment.
         def clear(key:)
           key = hkey(key)[0] # extract only the first part of the key
           pool do |redis|

data/lib/gitlab/experiment/cache.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'active_support/cache'
-
 module Gitlab
   class Experiment
     module Cache
@@ -21,7 +19,7 @@ module Gitlab
         end
 
         def write(value = nil)
-          store.write(key, value || @experiment.variant.name)
+          store.write(key, value || @experiment.assigned.name)
         end
 
         def delete
@@ -68,10 +66,8 @@ module Gitlab
 
           store.write(cache_key, value)
           store.delete(old_key)
-          return value
-        end
-
-        store.fetch(cache_key, &block)
+          break value
+        end || store.fetch(cache_key, &block)
       end
     end
   end

data/lib/gitlab/experiment/callbacks.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'active_support/callbacks'
-
 module Gitlab
   class Experiment
     module Callbacks
@@ -9,15 +7,85 @@ module Gitlab
       include ActiveSupport::Callbacks
 
       included do
-        define_callbacks(:unsegmented)
-        define_callbacks(:segmentation_check)
+        # Callbacks are listed in order of when they're executed when running an experiment.
+
+        # Exclusion check chain:
+        #
+        # The :exclusion_check chain is executed when determining if the context should be excluded from the experiment.
+        #
+        # If any callback returns true, further chain execution is terminated, the context will be considered excluded,
+        # and the control behavior will be provided.
         define_callbacks(:exclusion_check, skip_after_callbacks_if_terminated: true)
+
+        # Segmentation chain:
+        #
+        # The :segmentation chain is executed when no variant has been explicitly provided, the experiment is enabled,
+        # and the context hasn't been excluded.
+        #
+        # If the :segmentation callback chain doesn't need to be executed, the :segmentation_skipped chain will be
+        # executed as the fallback.
+        #
+        # If any callback explicitly sets a variant, further chain execution is terminated.
+        define_callbacks(:segmentation)
+        define_callbacks(:segmentation_skipped)
+
+        # Run chain:
+        #
+        # The :run chain is executed when the experiment is enabled, and the context hasn't been excluded.
+        #
+        # If the :run callback chain doesn't need to be executed, the :run_skipped chain will be executed as the
+        # fallback.
+        define_callbacks(:run)
+        define_callbacks(:run_skipped)
       end
 
       class_methods do
+        def registered_behavior_callbacks
+          @_registered_behavior_callbacks ||= {}
+        end
+
         private
 
-        def build_callback(chain, filters, **options)
+        def build_behavior_callback(filters, variant, **options, &block)
+          if registered_behavior_callbacks[variant.to_s]
+            raise ExistingBehaviorError, "a behavior for the `#{variant}` variant has already been registered"
+          end
+
+          callback_behavior = "#{variant}_behavior".to_sym
+
+          # Register a the behavior so we can define the block later.
+          registered_behavior_callbacks[variant.to_s] = callback_behavior
+
+          # Add our block or default behavior method.
+          filters.push(block) if block.present?
+          filters.unshift(callback_behavior) if filters.empty?
+
+          # Define and build the callback that will set our result.
+          define_callbacks(callback_behavior)
+          build_callback(callback_behavior, *filters, **options) do |target, callback|
+            target.instance_variable_set(:@_behavior_callback_result, callback.call(target, nil))
+          end
+        end
+
+        def build_exclude_callback(filters, **options)
+          build_callback(:exclusion_check, *filters, **options) do |target, callback|
+            throw(:abort) if target.instance_variable_get(:@_excluded) || callback.call(target, nil) == true
+          end
+        end
+
+        def build_segment_callback(filters, variant, **options)
+          build_callback(:segmentation, *filters, **options) do |target, callback|
+            if target.instance_variable_get(:@_assigned_variant_name).nil? && callback.call(target, nil)
+              target.assigned(variant)
+            end
+          end
+        end
+
+        def build_run_callback(filters, **options)
+          set_callback(:run, *filters.compact, **options)
+        end
+
+        def build_callback(chain, *filters, **options)
           filters = filters.compact.map do |filter|
             result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
             ->(target) { yield(target, result_lambda) }
@@ -28,6 +96,30 @@ module Gitlab
           set_callback(chain, *filters, **options)
         end
       end
+
+      private
+
+      def exclusion_callback_chain
+        :exclusion_check
+      end
+
+      def segmentation_callback_chain
+        return :segmentation if @_assigned_variant_name.nil? && enabled? && !excluded?
+
+        :segmentation_skipped
+      end
+
+      def run_callback_chain
+        return :run if enabled? && !excluded?
+
+        :run_skipped
+      end
+
+      def registered_behavior_callbacks
+        self.class.registered_behavior_callbacks.transform_values do |callback_behavior|
+          -> { run_callbacks(callback_behavior) { @_behavior_callback_result } }
+        end
+      end
     end
   end
 end