gitlab-experiment 0.4.4 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b1a2f81e8680cffda06d116655200c055dbb74a42a52a7fa8adcb84c720fb01
-  data.tar.gz: 715ea6dd8494457c169ef3147ca2edf2fc6e4c63571801e1f5b22814182ebb56
+  metadata.gz: c838adc97d97a29963030a4a74617fc48d6facf937e41add14b318322ad9e80d
+  data.tar.gz: ea8cda6c8d4848b7be8f4437c3d2c5b605414bd9e053fe768175e248e85c6f48
 SHA512:
-  metadata.gz: 7eec246157542ff0897f9af9f3135272d4d7fea31049443519fb550d5ca9cb3ad2d073ae8594104256d353b8042132f7ae5cba399bb44c38d7d5332154d5b849
-  data.tar.gz: ff4145fbe06fad2de4ef6295f4a0715c71c9ef4165308452d9dacd0c02a0df94d2152d138a6945c583c68d6c5966e1bb1e200fd79a7d9668382ee7c1d68861ea
+  metadata.gz: 310ce3f0f80041fa52866082985b744b6853ade992e36bb95dd7e9c1ebf433c91cd949548a362336caa15d96854c9c6050b8462b7b907583ff2f316482896d9b
+  data.tar.gz: e2e47f252f92c74dfbed63b3e92aceccb3cdc8522ef1d64e0e6fda9050b4d955846c20b8c49af9dc288fbc6fe80090a6d36ee1b1d5cabcdd88a17ac01c226ece

data/README.md

@@ -15,7 +15,7 @@ When we discuss the behavior of this gem, we'll use terms like experiment, conte
 - `candidate` defines that there's one experimental code path.
 - `variant(s)` is used when more than one experimental code path exists.
 
-Candidate and variant are the same concept, but simplify how we speak about experimental paths.<br clear="all">
+Candidate and variant are the same concept, but simplify how we speak about experimental paths -- this concept is also widely referred to as the "experiment group".<br clear="all">
 
 [[_TOC_]]
 
@@ -100,6 +100,9 @@ Here are some examples of what you can introduce once you have a custom experime
 
 ```ruby
 class NotificationToggleExperiment < ApplicationExperiment
+  # Exclude any users that aren't me.
+  exclude :all_but_me
+
   # Segment any account less than 2 weeks old into the candidate, without
   # asking the variant resolver to decide which variant to provide.
   segment :account_age, variant: :candidate
@@ -107,17 +110,21 @@ class NotificationToggleExperiment < ApplicationExperiment
   # Define the default control behavior, which can be overridden at
   # experiment time.
   def control_behavior
-    render_toggle
+    # render_toggle
   end
 
   # Define the default candidate behavior, which can be overridden
   # at experiment time.
   def candidate_behavior
-    render_button
+    # render_button
   end
 
   private
 
+  def all_but_me
+    context.actor&.username == 'jejacks0n'
+  end
+
   def account_age
     context.actor && context.actor.created_at < 2.weeks.ago
   end
@@ -177,7 +184,7 @@ Generally, defining segmentation rules is a better way to approach routing into
 experiment(:notification_toggle, :no_interface, actor: user) do |e|
   e.use { render_toggle } # control
   e.try { render_button } # candidate
-  e.try(:no_interface) { no_interface! } # variant
+  e.try(:no_interface) { no_interface! } # no_interface variant
 end
 ```
 
@@ -222,6 +229,27 @@ When an experiment is run, the segmentation rules are executed in the order they
 
 This means that any user with the name `'jejacks0n'`, regardless of account age, will always be provided the experience as defined in "variant_one". 
 
+### Exclusion rules
+
+Exclusion rules are similar to segmentation rules, but are intended to determine if a context should even be considered as something we should track events towards. Exclusion means we don't care about the events in relation to the given context. 
+
+```ruby
+class NotificationToggleExperiment < ApplicationExperiment
+  exclude { context.actor.username != 'jejacks0n' }
+  exclude { context.actor.created_at < 2.weeks.ago }
+end
+```
+
+The previous examples will force all users with a username not matching `'jejacks0n'` and all newish users into the control. No events would be tracked for those.
+
+You may need to check exclusion in custom tracking logic, by checking `excluded?` or wrapping your code in a callback block:
+
+```ruby
+run_callbacks(:exclusion_check) do
+  track(:my_event, value: expensive_method_call)
+end
+```
+
 ### Return value
 
 By default the return value is a `Gitlab::Experiment` instance. In simple cases you may want only the results of the experiment though. You can call `run` within the block to get the return value of the assigned variant.
@@ -341,7 +369,7 @@ Gitlab::Experiment.configure do |config|
 end
 ```
 
-More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb).
+More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb.tt).
 
 ### Client layer / JavaScript
 

data/lib/gitlab/experiment.rb

@@ -30,7 +30,7 @@ module Gitlab
         raise ArgumentError, 'name is required' if name.nil? && base?
 
         instance = constantize(name).new(name, variant_name, **context, &block)
-        return instance unless block_given?
+        return instance unless block
 
         instance.context.frozen? ? instance.run : instance.tap(&:run)
       end
@@ -58,16 +58,18 @@ module Gitlab
       raise ArgumentError, 'name is required' if name.blank? && self.class.base?
 
       @name = self.class.experiment_name(name, suffix: false)
-      @variant_name = variant_name
-      @excluded = []
       @context = Context.new(self, **context)
+      @variant_name = cache_variant(variant_name) { nil } if variant_name.present?
 
-      exclude { !@context.trackable? }
       compare { false }
 
       yield self if block_given?
     end
 
+    def inspect
+      "#<#{self.class.name || 'AnonymousClass'}:#{format('0x%016X', __id__)} @name=#{name} @signature=#{signature}>"
+    end
+
     def context(value = nil)
       return @context if value.blank?
 
@@ -81,32 +83,26 @@ module Gitlab
       end
 
       @variant_name = value unless value.blank?
-      @variant_name ||= :control if excluded?
 
-      @resolving_variant = true
-      resolved = :control
-      if (result = cache_variant(@variant_name) { resolve_variant_name }).present?
-        @variant_name = resolved = result.to_sym
+      if enabled?
+        @variant_name ||= :control if excluded?
+
+        @resolving_variant = true
+        if (result = cache_variant(@variant_name) { resolve_variant_name }).present?
+          @variant_name = result.to_sym
+        end
       end
 
-      Variant.new(name: resolved.to_s)
+      Variant.new(name: (@variant_name || :control).to_s)
     ensure
       @resolving_variant = false
     end
 
-    def exclude(&block)
-      @excluded << block
-    end
-
     def run(variant_name = nil)
       @result ||= begin
         variant_name = variant(variant_name).name
-        run_callbacks(variant_assigned? ? :unsegmented_run : :segmented_run) do
-          if respond_to?((behavior_name = "#{variant_name}_behavior"))
-            behaviors[variant_name] ||= -> { send(behavior_name) } # rubocop:disable GitlabSecurity/PublicSend
-          end
-
-          super(@variant_name = variant_name)
+        run_callbacks(run_with_segmenting? ? :segmented_run : :unsegmented_run) do
+          super(@variant_name ||= variant_name)
         end
       end
     end
@@ -116,7 +112,7 @@ module Gitlab
     end
 
     def track(action, **event_args)
-      return if excluded?
+      return unless should_track?
 
       instance_exec(action, event_args, &Configuration.tracking_behavior)
     end
@@ -129,20 +125,22 @@ module Gitlab
       @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
     end
 
-    def signature
-      { variant: variant.name, experiment: name }.merge(context.signature)
-    end
+    def behaviors
+      @behaviors ||= public_methods.each_with_object(super) do |name, behaviors|
+        next unless name.end_with?('_behavior')
 
-    def enabled?
-      true
+        behavior_name = name.to_s.sub(/_behavior$/, '')
+        behaviors[behavior_name] ||= -> { send(name) } # rubocop:disable GitlabSecurity/PublicSend
+      end
     end
 
-    def excluded?
-      @excluded.any? { |exclude| exclude.call(self) }
+    def try(name = nil, &block)
+      name = (name || 'candidate').to_s
+      behaviors[name] = block
     end
 
-    def variant_assigned?
-      !@variant_name.nil?
+    def signature
+      { variant: variant.name, experiment: name }.merge(context.signature)
     end
 
     def id
@@ -154,12 +152,33 @@ module Gitlab
       "Experiment;#{id}"
     end
 
+    def enabled?
+      true
+    end
+
+    def excluded?
+      @excluded ||= !@context.trackable? || # adhere to DNT headers
+        !run_callbacks(:exclusion_check) { :not_excluded } # didn't pass exclusion check
+    end
+
+    def should_track?
+      enabled? && !excluded?
+    end
+
     def key_for(hash)
       instance_exec(hash, &Configuration.context_hash_strategy)
     end
 
     protected
 
+    def run_with_segmenting?
+      !variant_assigned? && enabled? && !excluded?
+    end
+
+    def variant_assigned?
+      !@variant_name.nil?
+    end
+
     def resolve_variant_name
       instance_exec(@variant_name, &Configuration.variant_resolver)
     end

data/lib/gitlab/experiment/callbacks.rb

@@ -7,31 +7,34 @@ module Gitlab
       include ActiveSupport::Callbacks
 
       included do
-        define_callbacks(
-          :unsegmented_run,
-          skip_after_callbacks_if_terminated: true
-        )
-
-        define_callbacks(
-          :segmented_run,
-          skip_after_callbacks_if_terminated: false,
-          terminator: lambda do |target, result_lambda|
-            result_lambda.call
-            target.variant_assigned?
-          end
-        )
+        define_callbacks(:unsegmented_run)
+        define_callbacks(:segmented_run)
+        define_callbacks(:exclusion_check, skip_after_callbacks_if_terminated: true)
       end
 
       class_methods do
+        def exclude(*filter_list, **options, &block)
+          filters = filter_list.unshift(block).compact.map do |filter|
+            result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
+            lambda do |target|
+              throw(:abort) if target.instance_variable_get(:'@excluded') || result_lambda.call(target, nil) == true
+            end
+          end
+
+          raise ArgumentError, 'no filters provided' if filters.empty?
+
+          set_callback(:exclusion_check, :before, *filters, options)
+        end
+
         def segment(*filter_list, variant:, **options, &block)
           filters = filter_list.unshift(block).compact.map do |filter|
             result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
-            ->(target) { target.variant(variant) if result_lambda.call(target, nil) }
+            ->(target) { target.variant(variant) if !target.variant_assigned? && result_lambda.call(target, nil) }
           end
 
           raise ArgumentError, 'no filters provided' if filters.empty?
 
-          set_callback(:segmented_run, :before, *filters, options, &block)
+          set_callback(:segmented_run, :before, *filters, options)
         end
       end
     end

data/lib/gitlab/experiment/context.rb

@@ -42,6 +42,14 @@ module Gitlab
         @signature ||= { key: @experiment.key_for(@value), migration_keys: migration_keys }.compact
       end
 
+      def method_missing(method_name, *)
+        @value.include?(method_name.to_sym) ? @value[method_name.to_sym] : super
+      end
+
+      def respond_to_missing?(method_name, *)
+        @value.include?(method_name.to_sym) ? true : super
+      end
+
       private
 
       def process_migrations(value)

data/lib/gitlab/experiment/rspec.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Experiment
+    module RSpecHelpers
+      def stub_experiments(experiments)
+        experiments.each do |name, variant|
+          variant = :control if variant == false
+          raise ArgumentError, 'variant must be a symbol or false' unless variant.is_a?(Symbol)
+
+          klass = Gitlab::Experiment.send(:constantize, name) # rubocop:disable GitlabSecurity/PublicSend
+
+          # We have to use this high level any_instance behavior as there's
+          # not an alternative that allows multiple wrappings of `new`.
+          allow_any_instance_of(klass).to receive(:enabled?).and_return(true)
+          allow_any_instance_of(klass).to receive(:resolve_variant_name).and_return(variant.to_s)
+        end
+      end
+
+      def wrapped_experiment(experiment, shallow: false, failure: nil, &block)
+        if shallow
+          yield experiment if block.present?
+          return experiment
+        end
+
+        receive_wrapped_new = receive(:new).and_wrap_original do |new, *new_args, &new_block|
+          instance = new.call(*new_args)
+          instance.tap(&block) if block.present?
+          instance.tap(&new_block) if new_block.present?
+          instance
+        end
+
+        klass = experiment.class == Class ? experiment : experiment.class
+        if failure
+          expect(klass).to receive_wrapped_new, failure
+        else
+          allow(klass).to receive_wrapped_new
+        end
+      end
+    end
+
+    module RSpecMatchers
+      extend RSpec::Matchers::DSL
+
+      def require_experiment(experiment, matcher_name, classes: false)
+        klass = experiment.class == Class ? experiment : experiment.class
+        unless klass <= Gitlab::Experiment
+          raise(
+            ArgumentError,
+            "#{matcher_name} matcher is limited to experiment instances#{classes ? ' and classes' : ''}"
+          )
+        end
+
+        if experiment == klass && !classes
+          raise ArgumentError, "#{matcher_name} matcher requires an instance of an experiment"
+        end
+
+        experiment != klass
+      end
+
+      matcher :exclude do |context|
+        ivar = :'@excluded'
+
+        match do |experiment|
+          require_experiment(experiment, 'exclude')
+          experiment.context(context)
+
+          experiment.instance_variable_set(ivar, nil)
+          !experiment.run_callbacks(:exclusion_check) { :not_excluded }
+        end
+
+        failure_message do
+          %(expected #{context} to be excluded)
+        end
+
+        failure_message_when_negated do
+          %(expected #{context} not to be excluded)
+        end
+      end
+
+      matcher :segment do |context|
+        ivar = :'@variant_name'
+
+        match do |experiment|
+          require_experiment(experiment, 'segment')
+          experiment.context(context)
+
+          experiment.instance_variable_set(ivar, nil)
+          experiment.run_callbacks(:segmented_run)
+
+          @actual = experiment.instance_variable_get(ivar)
+          @expected ? @actual.to_s == @expected.to_s : @actual.present?
+        end
+
+        chain :into do |expected|
+          raise ArgumentError, 'variant name must be provided' if expected.blank?
+
+          @expected = expected.to_s
+        end
+
+        failure_message do
+          %(expected #{context} to be segmented#{message_details})
+        end
+
+        failure_message_when_negated do
+          %(expected #{context} not to be segmented#{message_details})
+        end
+
+        def message_details
+          message = ''
+          message += %( into variant\n    expected variant: #{@expected}) if @expected
+          message += %(\n      actual variant: #{@actual}) if @actual
+          message
+        end
+      end
+
+      matcher :track do |event, *event_args|
+        match do |experiment|
+          expect_tracking_on(experiment, false, event, *event_args)
+        end
+
+        match_when_negated do |experiment|
+          expect_tracking_on(experiment, true, event, *event_args)
+        end
+
+        chain :for do |expected_variant|
+          raise ArgumentError, 'variant name must be provided' if expected.blank?
+
+          @expected_variant = expected_variant.to_s
+        end
+
+        chain(:with_context) { |expected_context| @expected_context = expected_context }
+        chain(:on_any_instance) { @on_self = false }
+
+        def expect_tracking_on(experiment, negated, event, *event_args)
+          @experiment = experiment
+          @on_self = true if require_experiment(experiment, 'track', classes: !@on_self) && @on_self.nil?
+          wrapped_experiment(experiment, shallow: @on_self, failure: failure_message(:no_new, event)) do |instance|
+            @experiment = instance
+            allow(@experiment).to receive(:track)
+
+            if negated
+              expect(@experiment).not_to receive_tracking_call_for(event, *event_args)
+            else
+              expect(@experiment).to receive_tracking_call_for(event, *event_args)
+            end
+          end
+        end
+
+        def receive_tracking_call_for(event, *event_args)
+          receive(:track).with(*[event, *event_args]) do # rubocop:disable CodeReuse/ActiveRecord
+            if @expected_variant
+              expect(@experiment.variant.name).to eq(@expected_variant), failure_message(:variant, event)
+            end
+
+            if @expected_context
+              expect(@experiment.context.value).to include(@expected_context), failure_message(:context, event)
+            end
+          end
+        end
+
+        def failure_message(failure_type, event)
+          case failure_type
+          when :variant
+            <<~MESSAGE.strip
+              expected #{@experiment.inspect} to have tracked #{event.inspect} for variant
+                  expected variant: #{@expected_variant}
+                    actual variant: #{@experiment.variant.name}
+            MESSAGE
+          when :context
+            <<~MESSAGE.strip
+              expected #{@experiment.inspect} to have tracked #{event.inspect} with context
+                  expected context: #{@expected_context}
+                    actual context: #{@experiment.context.value}
+            MESSAGE
+          when :no_new
+            %(expected #{@experiment.inspect} to have tracked #{event.inspect}, but no new instances were created)
+          end
+        end
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Gitlab::Experiment::RSpecHelpers
+  config.include Gitlab::Experiment::Dsl
+
+  config.include Gitlab::Experiment::RSpecMatchers, :experiment
+  config.define_derived_metadata(file_path: Regexp.new('/spec/experiments/')) do |metadata|
+    metadata[:type] = :experiment
+  end
+end

data/lib/gitlab/experiment/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   class Experiment
-    VERSION = '0.4.4'
+    VERSION = '0.4.9'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-experiment
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.9
 platform: ruby
 authors:
 - GitLab
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-15 00:00:00.000000000 Z
+date: 2021-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -72,6 +72,7 @@ files:
 - lib/gitlab/experiment/cookies.rb
 - lib/gitlab/experiment/dsl.rb
 - lib/gitlab/experiment/engine.rb
+- lib/gitlab/experiment/rspec.rb
 - lib/gitlab/experiment/variant.rb
 - lib/gitlab/experiment/version.rb
 homepage: https://gitlab.com/gitlab-org/gitlab-experiment