gitlab-experiment 0.4.2 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 865d51c081670824fc7ffb48d4bc764fd5370606d3aa63b2196421b5570506e7
-  data.tar.gz: a007f872a3d56ee81c9925117f603e4930d1b7d689078edb074a2ba5567833f2
+  metadata.gz: a26425622afb7aa3f341ff403192963e276876b65902237018f852117168a19a
+  data.tar.gz: 0a5238c73a4b0217810b4f8161df6452b683b21a6573f1b657517302cef9cef0
 SHA512:
-  metadata.gz: 03d401a71b952b74519a21fa851a87ee9e104ccd19011af37ac41c36bec5fd63db3e68c62c05beb34ab296d354e00e15f013dd34a547c3b4c5c94474b89096cc
-  data.tar.gz: 2bb0260e40d4689446c1317273d85140fae9f56394821fda96c7bbc5d1acbe441f3e4024bf00ec05812f64f5588897137fc8061e790d871c5719742f6cedf516
+  metadata.gz: 567ca93e3f931c167bc1766490b07e1535db72e3b5da5986715412746354945ed799dab4c075e91c5015aa922bf5d7e5a82bdc940808aa8100c8897d5ee623b7
+  data.tar.gz: 3f053ed2ad8bdfc2201aacaef9c723332fba260f4f4cb54c65a92539d74ec100297540c34c97579c99723f0d98e98dc57cbe383a0731128d623b0a58b4786d7c

data/README.md

@@ -15,7 +15,7 @@ When we discuss the behavior of this gem, we'll use terms like experiment, conte
 - `candidate` defines that there's one experimental code path.
 - `variant(s)` is used when more than one experimental code path exists.
 
-Candidate and variant are the same concept, but simplify how we speak about experimental paths.<br clear="all">
+Candidate and variant are the same concept, but simplify how we speak about experimental paths -- this concept is also widely referred to as the "experiment group".<br clear="all">
 
 [[_TOC_]]
 
@@ -100,6 +100,9 @@ Here are some examples of what you can introduce once you have a custom experime
 
 ```ruby
 class NotificationToggleExperiment < ApplicationExperiment
+  # Exclude any users that aren't me.
+  exclude :all_but_me
+
   # Segment any account less than 2 weeks old into the candidate, without
   # asking the variant resolver to decide which variant to provide.
   segment :account_age, variant: :candidate
@@ -107,17 +110,21 @@ class NotificationToggleExperiment < ApplicationExperiment
   # Define the default control behavior, which can be overridden at
   # experiment time.
   def control_behavior
-    render_toggle
+    # render_toggle
   end
 
   # Define the default candidate behavior, which can be overridden
   # at experiment time.
   def candidate_behavior
-    render_button
+    # render_button
   end
 
   private
 
+  def all_but_me
+    context.actor&.username == 'jejacks0n'
+  end
+
   def account_age
     context.actor && context.actor.created_at < 2.weeks.ago
   end
@@ -177,7 +184,7 @@ Generally, defining segmentation rules is a better way to approach routing into
 experiment(:notification_toggle, :no_interface, actor: user) do |e|
   e.use { render_toggle } # control
   e.try { render_button } # candidate
-  e.try(:no_interface) { no_interface! } # variant
+  e.try(:no_interface) { no_interface! } # no_interface variant
 end
 ```
 
@@ -222,6 +229,27 @@ When an experiment is run, the segmentation rules are executed in the order they
 
 This means that any user with the name `'jejacks0n'`, regardless of account age, will always be provided the experience as defined in "variant_one". 
 
+### Exclusion rules
+
+Exclusion rules are similar to segmentation rules, but are intended to determine if a context should even be considered as something we should track events towards. Exclusion means we don't care about the events in relation to the given context. 
+
+```ruby
+class NotificationToggleExperiment < ApplicationExperiment
+  exclude { context.actor.username != 'jejacks0n' }
+  exclude { context.actor.created_at < 2.weeks.ago }
+end
+```
+
+The previous examples will force all users with a username not matching `'jejacks0n'` and all newish users into the control. No events would be tracked for those.
+
+You may need to check exclusion in custom tracking logic, by checking `excluded?` or wrapping your code in a callback block:
+
+```ruby
+run_callbacks(:exclusion_check) do
+  track(:my_event, value: expensive_method_call)
+end
+```
+
 ### Return value
 
 By default the return value is a `Gitlab::Experiment` instance. In simple cases you may want only the results of the experiment though. You can call `run` within the block to get the return value of the assigned variant.
@@ -341,7 +369,7 @@ Gitlab::Experiment.configure do |config|
 end
 ```
 
-More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb).
+More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb.tt).
 
 ### Client layer / JavaScript
 
@@ -388,7 +416,7 @@ Each of these approaches could be desirable given the objectives of your experim
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies.
-Then, run `bundle exec rspec` to run the tests. You can also run `bundle exec pry` for an
+Then, run `bundle exec rake` to run the tests. You can also run `bundle exec pry` for an
 interactive prompt that will allow you to experiment.
 
 ## Contributing

data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt

@@ -25,17 +25,17 @@ Gitlab::Experiment.configure do |config|
   # Logic this project uses to resolve a variant for a given experiment.
   #
   # Should return a symbol or string that represents the variant that should
-  # be assigned.
+  # be assigned. Blank or nil values will be defaulted to the control.
   #
   # This block is executed within the scope of the experiment and so can access
   # experiment methods, like `name`, `context`, and `signature`.
   config.variant_resolver = lambda do |requested_variant|
     # Run the control, unless a variant was requested in code:
-    requested_variant || 'control'
+    requested_variant
 
     # Run the candidate, unless a variant was requested, with a fallback:
     #
-    # requested_variant || variant_names.first || 'control'
+    # requested_variant || variant_names.first || nil
   end
 
   # Tracking behavior can be implemented to link an event to an experiment.

data/lib/gitlab/experiment.rb

@@ -30,7 +30,7 @@ module Gitlab
         raise ArgumentError, 'name is required' if name.nil? && base?
 
         instance = constantize(name).new(name, variant_name, **context, &block)
-        return instance unless block_given?
+        return instance unless block
 
         instance.context.frozen? ? instance.run : instance.tap(&:run)
       end
@@ -58,11 +58,9 @@ module Gitlab
       raise ArgumentError, 'name is required' if name.blank? && self.class.base?
 
       @name = self.class.experiment_name(name, suffix: false)
-      @variant_name = variant_name
-      @excluded = []
-      @context = Context.new(self, context)
+      @context = Context.new(self, **context)
+      @variant_name = cache_variant(variant_name) { nil } if variant_name.present?
 
-      exclude { !@context.trackable? }
       compare { false }
 
       yield self if block_given?
@@ -76,26 +74,31 @@ module Gitlab
     end
 
     def variant(value = nil)
+      if value.blank? && @variant_name || @resolving_variant
+        return Variant.new(name: (@variant_name || :unresolved).to_s)
+      end
+
       @variant_name = value unless value.blank?
-      @variant_name ||= :control if excluded?
 
-      resolved = cache { resolve_variant_name }
-      Variant.new(name: (resolved.presence || :control).to_s)
-    end
+      if enabled?
+        @variant_name ||= :control if excluded?
 
-    def exclude(&block)
-      @excluded << block
+        @resolving_variant = true
+        if (result = cache_variant(@variant_name) { resolve_variant_name }).present?
+          @variant_name = result.to_sym
+        end
+      end
+
+      Variant.new(name: (@variant_name || :control).to_s)
+    ensure
+      @resolving_variant = false
     end
 
     def run(variant_name = nil)
       @result ||= begin
         variant_name = variant(variant_name).name
-        run_callbacks(variant_assigned? ? :unsegmented_run : :segmented_run) do
-          if respond_to?((behavior_name = "#{variant_name}_behavior"))
-            behaviors[variant_name] ||= -> { send(behavior_name) } # rubocop:disable GitlabSecurity/PublicSend
-          end
-
-          super(@variant_name = variant_name)
+        run_callbacks(run_with_segmenting? ? :segmented_run : :unsegmented_run) do
+          super(@variant_name ||= variant_name)
         end
       end
     end
@@ -105,7 +108,7 @@ module Gitlab
     end
 
     def track(action, **event_args)
-      return if excluded?
+      return unless should_track?
 
       instance_exec(action, event_args, &Configuration.tracking_behavior)
     end
@@ -118,20 +121,22 @@ module Gitlab
       @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
     end
 
-    def signature
-      { variant: variant.name, experiment: name }.merge(context.signature)
-    end
+    def behaviors
+      @behaviors ||= public_methods.each_with_object(super) do |name, behaviors|
+        next unless name.end_with?('_behavior')
 
-    def enabled?
-      true
+        behavior_name = name.to_s.sub(/_behavior$/, '')
+        behaviors[behavior_name] ||= -> { send(name) } # rubocop:disable GitlabSecurity/PublicSend
+      end
     end
 
-    def excluded?
-      @excluded.any? { |exclude| exclude.call(self) }
+    def try(name = nil, &block)
+      name = (name || 'candidate').to_s
+      behaviors[name] = block
     end
 
-    def variant_assigned?
-      !@variant_name.nil?
+    def signature
+      { variant: variant.name, experiment: name }.merge(context.signature)
     end
 
     def id
@@ -143,19 +148,35 @@ module Gitlab
       "Experiment;#{id}"
     end
 
+    def enabled?
+      true
+    end
+
+    def excluded?
+      @excluded ||= !@context.trackable? || # adhere to DNT headers
+        !run_callbacks(:exclusion_check) { :not_excluded } # didn't pass exclusion check
+    end
+
+    def should_track?
+      enabled? && !excluded?
+    end
+
     def key_for(hash)
       instance_exec(hash, &Configuration.context_hash_strategy)
     end
 
     protected
 
-    def resolve_variant_name
-      return :unresolved if @resolving
+    def run_with_segmenting?
+      !variant_assigned? && enabled? && !excluded?
+    end
+
+    def variant_assigned?
+      !@variant_name.nil?
+    end
 
-      @resolving = true
-      result = instance_exec(@variant_name, &Configuration.variant_resolver)
-      @resolving = false
-      result
+    def resolve_variant_name
+      instance_exec(@variant_name, &Configuration.variant_resolver)
     end
 
     def generate_result(variant_name)

data/lib/gitlab/experiment/caching.rb

@@ -3,20 +3,26 @@
 module Gitlab
   class Experiment
     module Caching
-      def cache(&block)
-        return yield unless (cache = Configuration.cache)
+      def cache_variant(specified = nil, &block)
+        cache = Configuration.cache
+        return (specified.presence || yield) unless cache
 
         key, migrations = cache_strategy
-        migrated_cache(cache, migrations || [], key) or cache.fetch(key, &block)
+        result = migrated_cache(cache, migrations || [], key) || cache.fetch(key, &block)
+        return result unless specified.present?
+
+        cache.write(cache_key, specified) if result != specified
+        specified
+      end
+
+      def cache_key(key = nil)
+        "#{name}:#{key || context.signature[:key]}"
       end
 
       private
 
       def cache_strategy
-        [
-          "#{name}:#{context.signature[:key]}",
-          context.signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
-        ]
+        [cache_key, context.signature[:migration_keys]&.map { |key| cache_key(key) }]
       end
 
       def migrated_cache(cache, migrations, new_key)

data/lib/gitlab/experiment/callbacks.rb

@@ -7,31 +7,34 @@ module Gitlab
       include ActiveSupport::Callbacks
 
       included do
-        define_callbacks(
-          :unsegmented_run,
-          skip_after_callbacks_if_terminated: true
-        )
-
-        define_callbacks(
-          :segmented_run,
-          skip_after_callbacks_if_terminated: false,
-          terminator: lambda do |target, result_lambda|
-            result_lambda.call
-            target.variant_assigned?
-          end
-        )
+        define_callbacks(:unsegmented_run)
+        define_callbacks(:segmented_run)
+        define_callbacks(:exclusion_check, skip_after_callbacks_if_terminated: true)
       end
 
       class_methods do
+        def exclude(*filter_list, **options, &block)
+          filters = filter_list.unshift(block).compact.map do |filter|
+            result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
+            lambda do |target|
+              throw(:abort) if target.instance_variable_get(:'@excluded') || result_lambda.call(target, nil) == true
+            end
+          end
+
+          raise ArgumentError, 'no filters provided' if filters.empty?
+
+          set_callback(:exclusion_check, :before, *filters, options)
+        end
+
         def segment(*filter_list, variant:, **options, &block)
           filters = filter_list.unshift(block).compact.map do |filter|
             result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
-            ->(target) { target.variant(variant) if result_lambda.call(target, nil) }
+            ->(target) { target.variant(variant) if !target.variant_assigned? && result_lambda.call(target, nil) }
           end
 
           raise ArgumentError, 'no filters provided' if filters.empty?
 
-          set_callback(:segmented_run, :before, *filters, options, &block)
+          set_callback(:segmented_run, :before, *filters, options)
         end
       end
     end

data/lib/gitlab/experiment/configuration.rb

@@ -25,8 +25,9 @@ module Gitlab
       @cookie_domain = :all
 
       # Logic this project uses to resolve a variant for a given experiment.
+      # If no variant is determined, the control will be used.
       @variant_resolver = lambda do |requested_variant|
-        requested_variant || :control
+        requested_variant
       end
 
       # Tracking behavior can be implemented to link an event to an experiment.

data/lib/gitlab/experiment/context.rb

@@ -42,6 +42,14 @@ module Gitlab
         @signature ||= { key: @experiment.key_for(@value), migration_keys: migration_keys }.compact
       end
 
+      def method_missing(method_name, *)
+        @value.include?(method_name.to_sym) ? @value[method_name.to_sym] : super
+      end
+
+      def respond_to_missing?(method_name, *)
+        @value.include?(method_name.to_sym) ? true : super
+      end
+
       private
 
       def process_migrations(value)

data/lib/gitlab/experiment/cookies.rb

@@ -11,7 +11,7 @@ module Gitlab
         return hash if cookie_jar.nil?
 
         resolver = [hash, :actor, cookie_name, cookie_jar.signed[cookie_name]]
-        resolve_cookie(*resolver) or generate_cookie(*resolver)
+        resolve_cookie(*resolver) || generate_cookie(*resolver)
       end
 
       def cookie_jar

data/lib/gitlab/experiment/rspec.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Experiment
+    module RSpecHelpers
+      def stub_experiments(experiments)
+        experiments.each do |name, variant|
+          variant = :control if variant == false
+          raise ArgumentError, 'variant must be a symbol or false' unless variant.is_a?(Symbol)
+
+          klass = Gitlab::Experiment.send(:constantize, name) # rubocop:disable GitlabSecurity/PublicSend
+          allow(klass).to receive(:new).and_wrap_original do |new, *args, &block|
+            new.call(*args).tap do |instance|
+              allow(instance).to receive(:enabled?).and_return(true)
+              allow(instance).to receive(:resolve_variant_name).and_return(variant.to_s)
+
+              block.call(instance) if block.present?
+            end
+          end
+        end
+      end
+    end
+
+    module RSpecMatchers
+      extend RSpec::Matchers::DSL
+
+      matcher :exclude do |context|
+        ivar = :'@excluded'
+
+        match do |experiment|
+          experiment.context(context)
+
+          experiment.instance_variable_set(ivar, nil)
+          !experiment.run_callbacks(:exclusion_check) { :not_excluded }
+        end
+
+        failure_message do
+          %(expected #{context} to be excluded)
+        end
+
+        failure_message_when_negated do
+          %(expected #{context} not to be excluded)
+        end
+      end
+
+      matcher :segment do |context|
+        ivar = :'@variant_name'
+
+        match do |experiment|
+          experiment.context(context)
+
+          experiment.instance_variable_set(ivar, nil)
+          experiment.run_callbacks(:segmented_run)
+
+          @actual = experiment.instance_variable_get(ivar)
+          @expected ? @actual.to_s == @expected.to_s : @actual.present?
+        end
+
+        chain :into do |expected|
+          raise ArgumentError, 'variant name must be provided' if expected.blank?
+
+          @expected = expected.to_s
+        end
+
+        failure_message do
+          %(expected #{context} to be segmented#{message_details})
+        end
+
+        failure_message_when_negated do
+          %(expected #{context} not to be segmented#{message_details})
+        end
+
+        def message_details
+          message = ''
+          message += %(\n    into: #{@expected}) if @expected
+          message += %(\n  actual: #{@actual}) if @actual
+          message
+        end
+      end
+
+      matcher :track do |event, *event_args|
+        match do |experiment|
+          wrapped(experiment) { |instance| expect(instance).to receive_with(event, *event_args) }
+        end
+
+        match_when_negated do |experiment|
+          wrapped(experiment) { |instance| expect(instance).not_to receive_with(event, *event_args) }
+        end
+
+        def wrapped(experiment, &block)
+          allow(experiment.class).to receive(:new).and_wrap_original do |new, *new_args|
+            new.call(*new_args).tap(&block)
+          end
+        end
+
+        def receive_with(*args)
+          receive(:track).with(*args) # rubocop:disable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Gitlab::Experiment::RSpecHelpers
+
+  config.include Gitlab::Experiment::RSpecMatchers, :experiment
+  config.define_derived_metadata(file_path: %r{/spec/experiments/}) do |metadata|
+    metadata[:type] = :experiment
+  end
+end

data/lib/gitlab/experiment/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   class Experiment
-    VERSION = '0.4.2'
+    VERSION = '0.4.7'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-experiment
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.7
 platform: ruby
 authors:
 - GitLab
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-19 00:00:00.000000000 Z
+date: 2021-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -72,6 +72,7 @@ files:
 - lib/gitlab/experiment/cookies.rb
 - lib/gitlab/experiment/dsl.rb
 - lib/gitlab/experiment/engine.rb
+- lib/gitlab/experiment/rspec.rb
 - lib/gitlab/experiment/variant.rb
 - lib/gitlab/experiment/version.rb
 homepage: https://gitlab.com/gitlab-org/gitlab-experiment