gitlab-experiment 0.6.5 → 0.7.0

data/lib/gitlab/experiment/configuration.rb

@@ -4,86 +4,231 @@ require 'singleton'
 require 'logger'
 require 'digest'
 
-require 'active_support/deprecation'
-
 module Gitlab
   class Experiment
     class Configuration
       include Singleton
 
-      # Prefix all experiment names with a given value. Use `nil` for none.
+      # Prefix all experiment names with a given string value.
+      # Use `nil` for no prefix.
       @name_prefix = nil
 
-      # The logger is used to log various details of the experiments.
+      # The logger can be used to log various details of the experiments.
       @logger = Logger.new($stdout)
 
       # The base class that should be instantiated for basic experiments.
+      # It should be a string, so we can constantize it later.
       @base_class = 'Gitlab::Experiment'
 
-      # The caching layer is expected to respond to fetch, like Rails.cache.
+      # Require experiments to be defined in a class, with variants registered.
+      # This will disallow any anonymous experiments that are run inline
+      # without previously defining a class.
+      @strict_registration = false
+
+      # The caching layer is expected to match the Rails.cache interface.
+      # If no cache is provided some rollout strategies may behave differently.
+      # Use `nil` for no caching.
       @cache = nil
 
       # The domain to use on cookies.
+      #
+      # When not set, it uses the current host. If you want to provide specific
+      # hosts, you use `:all`, or provide an array.
+      #
+      # Examples:
+      #   nil, :all, or ['www.gitlab.com', '.gitlab.com']
       @cookie_domain = :all
 
-      # The default rollout strategy only works for single variant experiments.
-      # It's expected that you use a more advanced rollout for multiple variant
-      # experiments.
-      @default_rollout = Rollout::Base.new
+      # The default rollout strategy.
+      #
+      # The recommended default rollout strategy when not using caching would
+      # be `Gitlab::Experiment::Rollout::Percent` as that will consistently
+      # assign the same variant with or without caching.
+      #
+      # Gitlab::Experiment::Rollout::Base can be inherited to implement your
+      # own rollout strategies.
+      #
+      # Each experiment can specify its own rollout strategy:
+      #
+      # class ExampleExperiment < ApplicationExperiment
+      #   default_rollout :random,              # :percent, :round_robin,
+      #                   include_control: true # or MyCustomRollout
+      # end
+      #
+      # Included rollout strategies:
+      #   Gitlab::Experiment::Rollout::Percent, (recommended)
+      #   Gitlab::Experiment::Rollout::RoundRobin, or
+      #   Gitlab::Experiment::Rollout::Random
+      @default_rollout = Gitlab::Experiment::Rollout::Percent.new(
+        include_control: true # include control in possible assignments
+      )
 
       # Secret seed used in generating context keys.
+      #
+      # You'll typically want to use an environment variable or secret value
+      # for this.
+      #
+      # Consider not using one that's shared with other systems, like Rails'
+      # SECRET_KEY_BASE for instance. Generate a new secret and utilize that
+      # instead.
       @context_key_secret = nil
 
-      # Bit length used by SHA2 in generating context keys - (256, 384 or 512.)
+      # Bit length used by SHA2 in generating context keys.
+      #
+      # Using a higher bit length would require more computation time.
+      #
+      # Valid bit lengths:
+      #   256, 384, or 512
       @context_key_bit_length = 256
 
       # The default base path that the middleware (or rails engine) will be
-      # mounted.
+      # mounted. The middleware enables an instrumentation url, that's similar
+      # to links that can be instrumented in email campaigns.
+      #
+      # Use `nil` if you don't want to mount the middleware.
+      #
+      # Examples:
+      #   '/-/experiment', '/redirect', nil
       @mount_at = nil
 
-      # The middleware won't redirect to urls that aren't considered valid.
-      # Expected to return a boolean value.
-      @redirect_url_validator = ->(_redirect_url) { true }
-
-      # Logic this project uses to determine inclusion in a given experiment.
+      # When using the middleware, links can be instrumented and redirected
+      # elsewhere. This can be exploited to make a harmful url look innocuous
+      # or that it's a valid url on your domain. To avoid this, you can provide
+      # your own logic for what urls will be considered valid and redirected
+      # to.
+      #
       # Expected to return a boolean value.
-      @inclusion_resolver = ->(_requested_variant) { false }
+      @redirect_url_validator = lambda do |_redirect_url|
+        true
+      end
 
       # Tracking behavior can be implemented to link an event to an experiment.
+      #
+      # This block is executed within the scope of the experiment and so can
+      # access experiment methods, like `name`, `context`, and `signature`.
       @tracking_behavior = lambda do |event, args|
+        # An example of using a generic logger to track events:
         Configuration.logger.info("#{self.class.name}[#{name}] #{event}: #{args.merge(signature: signature)}")
+
+        # Using something like snowplow to track events (in gitlab):
+        #
+        # Gitlab::Tracking.event(name, event, **args.merge(
+        #   context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
+        #     'iglu:com.gitlab/gitlab_experiment/jsonschema/0-2-0', signature
+        #   )
+        # ))
+      end
+
+      # Logic designed to respond when a given experiment is nested within
+      # another experiment. This can be useful to identify overlaps and when a
+      # code path leads to an experiment being nested within another.
+      #
+      # Reporting complexity can arise when one experiment changes rollout, and
+      # a downstream experiment is impacted by that.
+      #
+      # The base_class or a custom experiment can provide a `nest_experiment`
+      # method that implements its own logic that may allow certain experiments
+      # to be nested within it.
+      #
+      # This block is executed within the scope of the experiment and so can
+      # access experiment methods, like `name`, `context`, and `signature`.
+      #
+      # The default exception will include the where the experiment calls were
+      # initiated on, so for instance:
+      #
+      # Gitlab::Experiment::NestingError: unable to nest level2 within level1:
+      #   level1 initiated by file_name.rb:2
+      #   level2 initiated by file_name.rb:3
+      @nested_behavior = lambda do |nested_experiment|
+        raise NestingError.new(experiment: self, nested_experiment: nested_experiment)
       end
 
       # Called at the end of every experiment run, with the result.
+      #
+      # You may want to track that you've assigned a variant to a given
+      # context, or push the experiment into the client or publish results
+      # elsewhere like into redis.
+      #
+      # This block is executed within the scope of the experiment and so can
+      # access experiment methods, like `name`, `context`, and `signature`.
       @publishing_behavior = lambda do |_result|
+        # Track the event using our own configured tracking logic.
         track(:assignment)
+
+        # Log using our logging system, so the result (which can be large) can
+        # be reviewed later if we want to.
+        #
+        # Lograge::Event.log(experiment: name, result: result, signature: signature)
+
+        # Experiments that have been run during the request lifecycle can be
+        # pushed to the client layer by injecting the published experiments
+        # into javascript in a layout or view using something like:
+        #
+        # = javascript_tag(nonce: content_security_policy_nonce) do
+        #   window.experiments = #{raw Gitlab::Experiment.published_experiments.to_json};
       end
 
       class << self
-        # TODO: Added deprecation in release 0.6.0
+        # @deprecated
         def context_hash_strategy=(block)
-          ActiveSupport::Deprecation.warn('context_hash_strategy has been deprecated, instead configure' \
-            ' `context_key_secret` and `context_key_bit_length`.')
+          deprecated(
+            :context_hash_strategy,
+            'instead use `context_key_secret` and `context_key_bit_length`',
+            version: '0.7.0'
+          )
+
           @__context_hash_strategy = block
         end
 
-        # TODO: Added deprecation in release 0.5.0
+        # @deprecated
         def variant_resolver
-          ActiveSupport::Deprecation.warn('variant_resolver is deprecated, instead use `inclusion_resolver` with a' \
-            ' block that returns a boolean.')
-          @inclusion_resolver
+          deprecated(
+            :variant_resolver,
+            'instead use `inclusion_resolver` with a block that returns a boolean',
+            version: '0.6.5'
+          )
+
+          @__inclusion_resolver
         end
 
+        # @deprecated
         def variant_resolver=(block)
-          ActiveSupport::Deprecation.warn('variant_resolver is deprecated, instead use `inclusion_resolver` with a' \
-            ' block that returns a boolean.')
-          @inclusion_resolver = block
+          deprecated(
+            :variant_resolver,
+            'instead use `inclusion_resolver` with a block that returns a boolean',
+            version: '0.6.5'
+          )
+
+          @__inclusion_resolver = block
+        end
+
+        # @deprecated
+        def inclusion_resolver=(block)
+          deprecated(
+            :inclusion_resolver,
+            'instead put this logic into custom rollout strategies',
+            version: '0.7.0'
+          )
+
+          @__inclusion_resolver = block
+        end
+
+        # @deprecated
+        def inclusion_resolver
+          deprecated(
+            :inclusion_resolver,
+            'instead put this logic into custom rollout strategies',
+            version: '0.7.0'
+          )
+
+          @__inclusion_resolver
         end
 
         attr_accessor(
           :name_prefix,
           :logger,
           :base_class,
+          :strict_registration,
           :cache,
           :cookie_domain,
           :context_key_secret,
@@ -91,10 +236,33 @@ module Gitlab
           :mount_at,
           :default_rollout,
           :redirect_url_validator,
-          :inclusion_resolver,
           :tracking_behavior,
+          :nested_behavior,
           :publishing_behavior
         )
+
+        # Internal helpers warnings.
+        def deprecated(*args, version:, stack: 0)
+          deprecator = deprecator(version)
+          args << args.pop.to_s.gsub('{{release}}', "#{deprecator.gem_name} #{deprecator.deprecation_horizon}")
+          args << caller_locations(4 + stack)
+
+          if args.length == 2
+            deprecator.warn(*args)
+          else
+            args[0] = "`#{args[0]}`"
+            deprecator.deprecation_warning(*args)
+          end
+        end
+
+        private
+
+        def deprecator(version = VERSION)
+          version = Gem::Version.new(version).bump.to_s
+
+          @__dep_versions ||= {}
+          @__dep_versions[version] ||= ActiveSupport::Deprecation.new(version, 'Gitlab::Experiment')
+        end
       end
     end
   end

data/lib/gitlab/experiment/context.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'gitlab/experiment/cookies'
-
 module Gitlab
   class Experiment
     class Context

data/lib/gitlab/experiment/cookies.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'securerandom'
-
 module Gitlab
   class Experiment
     module Cookies

data/lib/gitlab/experiment/engine.rb

@@ -6,7 +6,8 @@ module Gitlab
   class Experiment
     include ActiveModel::Model
 
-    # used for generating routes
+    # Used for generating routes. We've included the method and `ActiveModel::Model` here because these things don't
+    # make sense outside of Rails environments.
     def self.model_name
       ActiveModel::Name.new(self, Gitlab)
     end

data/lib/gitlab/experiment/errors.rb

@@ -4,6 +4,26 @@ module Gitlab
   class Experiment
     Error = Class.new(StandardError)
     InvalidRolloutRules = Class.new(Error)
-    NestingError = Class.new(Error)
+    UnregisteredExperiment = Class.new(Error)
+    ExistingBehaviorError = Class.new(Error)
+    BehaviorMissingError = Class.new(Error)
+
+    class NestingError < Error
+      def initialize(experiment:, nested_experiment:)
+        messages = []
+        experiments = [nested_experiment, experiment]
+
+        callers = caller_locations
+        callers.select.with_index do |caller, index|
+          next if caller.label != 'experiment'
+
+          messages << "  #{experiments[messages.length].name} initiated by #{callers[index + 1]}"
+        end
+
+        messages << ["unable to nest #{nested_experiment.name} within #{experiment.name}:"]
+
+        super(messages.reverse.join("\n"))
+      end
+    end
   end
 end

data/lib/gitlab/experiment/nestable.rb

@@ -9,8 +9,8 @@ module Gitlab
         set_callback :run, :around, :manage_nested_stack
       end
 
-      def nest_experiment(other)
-        raise NestingError, "unable to nest the #{other.name} experiment within the #{name} experiment"
+      def nest_experiment(nested_experiment)
+        instance_exec(nested_experiment, &Configuration.nested_behavior)
       end
 
       private

data/lib/gitlab/experiment/rollout/percent.rb

@@ -1,30 +1,39 @@
 # frozen_string_literal: true
 
-require 'zlib'
-
+# The percent rollout strategy is the most comprehensive included with Gitlab::Experiment. It allows specifying the
+# percentages per variant using an array, a hash, or will default to even distribution when no rules are provided.
+#
+# A given experiment id (context key) will always be given the same variant assignment.
+#
+# Example configuration usage:
+#
+# config.default_rollout = Gitlab::Experiment::Rollout::Percent.new
+#
+# Example class usage:
+#
+# class PillColorExperiment < ApplicationExperiment
+#   control { }
+#   variant(:red) { }
+#   variant(:blue) { }
+#
+#   # Even distribution between all behaviors.
+#   default_rollout :percent
+#
+#   # With specific distribution percentages.
+#   default_rollout :percent, distribution: { control: 25, red: 30, blue: 45 }
+# end
+#
 module Gitlab
   class Experiment
     module Rollout
       class Percent < Base
-        def execute
-          crc = normalized_id
-          total = 0
-
-          case distribution_rules
-          # run through the rules until finding an acceptable one
-          when Array then variant_names[distribution_rules.find_index { |percent| crc % 100 <= total += percent }]
-          # run through the variant names until finding an acceptable one
-          when Hash then distribution_rules.find { |_, percent| crc % 100 <= total += percent }.first
-          # when there are no rules, assume even distribution
-          else variant_names[crc % variant_names.length]
-          end
-        end
+        protected
 
         def validate!
           case distribution_rules
           when nil then nil
           when Array, Hash
-            if distribution_rules.length != variant_names.length
+            if distribution_rules.length != behavior_names.length
               raise InvalidRolloutRules, "the distribution rules don't match the number of variants defined"
             end
           else
@@ -32,6 +41,20 @@ module Gitlab
           end
         end
 
+        def execute_assigment
+          crc = normalized_id
+          total = 0
+
+          case distribution_rules
+          when Array # run through the rules until finding an acceptable one
+            behavior_names[distribution_rules.find_index { |percent| crc % 100 <= total += percent }]
+          when Hash # run through the variant names until finding an acceptable one
+            distribution_rules.find { |_, percent| crc % 100 <= total += percent }.first
+          else # assume even distribution on no rules
+            behavior_names.empty? ? nil : behavior_names[crc % behavior_names.length]
+          end
+        end
+
         private
 
         def normalized_id
@@ -39,7 +62,7 @@ module Gitlab
         end
 
         def distribution_rules
-          @options[:distribution]
+          options[:distribution]
         end
       end
     end

data/lib/gitlab/experiment/rollout/random.rb

@@ -1,13 +1,34 @@
 # frozen_string_literal: true
 
+# The random rollout strategy will randomly assign a variant when the context is determined to be within the experiment
+# group.
+#
+# If caching is enabled this is a predicable and consistent assignment that will eventually assign a variant (since
+# control isn't cached) but if caching isn't enabled, assignment will be random each time.
+#
+# Example configuration usage:
+#
+# config.default_rollout = Gitlab::Experiment::Rollout::Random.new
+#
+# Example class usage:
+#
+# class PillColorExperiment < ApplicationExperiment
+#   control { }
+#   variant(:red) { }
+#   variant(:blue) { }
+#
+#   # Randomize between all behaviors, with a mostly even distribution).
+#   default_rollout :random
+# end
+#
 module Gitlab
   class Experiment
     module Rollout
       class Random < Base
-        # Pick a random variant if we're in the experiment group. It doesn't
-        # take into account small sample sizes but is useful and performant.
-        def execute
-          variant_names.sample
+        protected
+
+        def execute_assigment
+          behavior_names.sample # pick a random variant
         end
       end
     end

data/lib/gitlab/experiment/rollout/round_robin.rb

@@ -1,21 +1,38 @@
 # frozen_string_literal: true
 
+# The round robin strategy will assign the next variant in the list, looping back to the first variant after all
+# variants have been assigned. This is useful for very small sample sizes where very even distribution can be required.
+#
+# Requires a cache to be configured.
+#
+# Keeps track of the number of assignments into the experiment group, and uses this to rotate "round robin" style
+# through the variants that are defined.
+#
+# Example configuration usage:
+#
+# config.default_rollout = Gitlab::Experiment::Rollout::RoundRobin.new
+#
+# Example class usage:
+#
+# class PillColorExperiment < ApplicationExperiment
+#   control { }
+#   variant(:red) { }
+#   variant(:blue) { }
+#
+#   # Rotate evenly between all behaviors.
+#   default_rollout :round_robin
+# end
+#
 module Gitlab
   class Experiment
     module Rollout
       class RoundRobin < Base
         KEY_NAME = :last_round_robin_variant
 
-        # Requires a cache to be configured.
-        #
-        # Keeps track of the number of assignments into the experiment group,
-        # and uses this to rotate "round robin" style through the variants
-        # that are defined.
-        #
-        # Relatively performant, but requires a cache, and is dependent on the
-        # performance of that cache store.
-        def execute
-          variant_names[(cache.attr_inc(KEY_NAME) - 1) % variant_names.size]
+        protected
+
+        def execute_assigment
+          behavior_names[(cache.attr_inc(KEY_NAME) - 1) % behavior_names.size]
         end
       end
     end

data/lib/gitlab/experiment/rollout.rb

@@ -14,27 +14,63 @@ module Gitlab
       end
 
       class Base
-        attr_reader :experiment
+        DEFAULT_OPTIONS = {
+          include_control: false
+        }.freeze
+
+        attr_reader :experiment, :options
 
         delegate :variant_names, :cache, :id, to: :experiment
 
         def initialize(options = {})
-          @options = options
-          # validate! # we want to validate here, but we can't yet
+          @options = DEFAULT_OPTIONS.merge(options)
         end
 
-        def rollout_for(experiment)
+        def for(experiment)
+          raise ArgumentError, 'you must provide an experiment instance' unless experiment.class <= Gitlab::Experiment
+
           @experiment = experiment
-          validate! # until we have variant registration we can only validate here
-          execute
+
+          self
+        end
+
+        def enabled?
+          require_experiment(__method__)
+
+          true
+        end
+
+        def resolve
+          require_experiment(__method__)
+
+          return nil if @experiment.respond_to?(:experiment_group?) && !@experiment.experiment_group?
+
+          validate! # allow the rollout strategy to validate itself
+
+          assignment = execute_assigment
+          assignment == :control ? nil : assignment # avoid caching control
         end
 
+        protected
+
         def validate!
           # base is always valid
         end
 
-        def execute
-          variant_names.first
+        def execute_assigment
+          behavior_names.first
+        end
+
+        private
+
+        def require_experiment(method_name)
+          return if @experiment.present?
+
+          raise ArgumentError, "you need to call `for` with an experiment instance before chaining `#{method_name}`"
+        end
+
+        def behavior_names
+          options[:include_control] ? [:control] + variant_names : variant_names
         end
       end
     end