gitlab-experiment 0.4.0 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8524dff7f908e481923e3131f3dd67c34f426e31ed10171896ea10a209e0b74a
-  data.tar.gz: 151f2e7f7bbf9692350c02f46f0c4c8cfa6f9c7ec05fdcb0fda50e34324c06fb
+  metadata.gz: 81476752521c6ead83308324fdcf360db7b8182bab340c9ffe7ad4eec21b998e
+  data.tar.gz: 006d94ff92104bef820be7d6c7c9638b20a6e0a4a533439fca056b4cba31a0bc
 SHA512:
-  metadata.gz: 3f23698aabf3968c77f49cdb924125879663670c12d22c390a47f7c791687f87fb1211adf432777f66012e3be1db2e90d5c33a1dc0a156332bb8926938c4e587
-  data.tar.gz: 51973494136edef02672c9086970fec2ecb1fb3a6716edb10d1cfe758172f8d256ecd6ef1b5cefdd50133898637e022af527fc7e20df2212b1f7210db2dd087e
+  metadata.gz: a5644f3cf6798ddc27034b8857cf03bb77904b3b5c8a80b84c398134ad143b919e16ddc23746a58e63b71d0464c92ec5cce682ddb2039e471b55cfeb42e3cd4a
+  data.tar.gz: 2dd2adf62427f96a9745d9681ee87b5621acf43fc4dae33ffefdadfad5af6ec3a0ad1e79e601a4d6260e61fa64faf635af9ae753e3fb474e033af7d2d7d55381

data/README.md

@@ -1,4 +1,5 @@
-# GitLab Experiment
+GitLab Experiment
+=================
 
 <img alt="experiment" src="/uploads/60990b2dbf4c0406bbf8b7f998de2dea/experiment.png" align="right" width="40%">
 
@@ -16,6 +17,8 @@ When we discuss the behavior of this gem, we'll use terms like experiment, conte
 
 Candidate and variant are the same concept, but simplify how we speak about experimental paths.<br clear="all">
 
+[[_TOC_]]
+
 ## Installation
 
 Add the gem to your Gemfile and then `bundle install`.
@@ -24,7 +27,7 @@ Add the gem to your Gemfile and then `bundle install`.
 gem 'gitlab-experiment'
 ```
 
-If you're using Rails, you can install the initializer. It provides basic configuration and documentation.
+If you're using Rails, you can install the initializer which provides basic configuration, documentation, and the base experiment class that all your experiments can inherit from.
 
 ```shell
 $ rails generate gitlab:experiment:install
@@ -79,72 +82,84 @@ To this end, we track events that are important by calling the same experiment e
 experiment(:notification_toggle, actor: user).track(:clicked_button)
 ```
 
-<details>
-  <summary>You can also use the more low level class or instance interfaces...</summary>
+### Custom experiments
 
-### Class level interface using `.run`
+You can craft more advanced behaviors by defining custom experiments at a higher level. To do this you can define a class that inherits from `ApplicationExperiment` (or `Gitlab::Experiment`).
 
-```ruby
-exp = Gitlab::Experiment.run(:notification_toggle, actor: user) do |e|
-  # Context may be passed in the block, but must be finalized before calling
-  # run or track.
-  e.context(project: project) # add the project to the context
+Let's say you want to do more advanced segmentation, or provide default behavior for the variants on the experiment we've already outlined above -- that way if the variants aren't defined in the block at the time the experiment is run, these methods will be used. 
 
-  # Define the control and candidate variant.
-  e.use { render_toggle } # control
-  e.try { render_button } # candidate
-end
+You can generate a custom experiment by running:
 
-# Track an event on the experiment we've defined.
-exp.track(:clicked_button)
-```
+```shell
+$ rails generate gitlab:experiment NotificationToggle control candidate
+``` 
+
+This will generate a file in `app/experiments/notification_toggle_experiment.rb`, as well as a test file for you to further expand on.
 
-### Instance level interface
+Here are some examples of what you can introduce once you have a custom experiment defined.
 
 ```ruby
-exp = Gitlab::Experiment.new(:notification_toggle, actor: user)
-# Additional context may be provided to the instance (exp) but must be
-# finalized before calling run or track.
-exp.context(project: project) # add the project id to the context
+class NotificationToggleExperiment < ApplicationExperiment
+  # Segment any account less than 2 weeks old into the candidate, without
+  # asking the variant resolver to decide which variant to provide.
+  segment :account_age, variant: :candidate
+
+  # Define the default control behavior, which can be overridden at
+  # experiment time.
+  def control_behavior
+    render_toggle
+  end
+
+  # Define the default candidate behavior, which can be overridden
+  # at experiment time.
+  def candidate_behavior
+    render_button
+  end
+
+  private
 
-# Define the control and candidate variant.
-exp.use { render_toggle } # control
-exp.try { render_button } # candidate
+  def account_age
+    context.actor && context.actor.created_at < 2.weeks.ago
+  end
+end
 
-# Run the experiment, returning the result.
+# The class will be looked up based on the experiment name provided. 
+exp = experiment(:notification_toggle, actor: user)
+exp # => instance of NotificationToggleExperiment
+
+# Run the experiment -- returning the result.
 exp.run
 
 # Track an event on the experiment we've defined.
 exp.track(:clicked_button)
 ```
 
-</details>
+You can now also do things very similar to the simple examples and override the default variant behaviors defined in the custom experiment -- keeping in mind that this should be carefully considered within the scope of your experiment.
+
+```ruby
+experiment(:notification_toggle, actor: user) do |e|
+  e.use { render_special_toggle } # override default control behavior
+end
+```
 
 <details>
-  <summary>You can define and use custom classes...</summary>
+  <summary>You can also use the lower level class interface...</summary>
 
-### Custom class
+### Using the `.run` approach
 
-```ruby
-class NotificationExperiment < Gitlab::Experiment
-  def initialize(variant_name = nil, **context, &block)
-    super(:notification_toggle, variant_name, **context, &block)
+This is useful if you haven't included the DSL and so don't have access to the `experiment` method, but still want to execute an experiment. This is ultimately what the `experiment` method calls through to, and the method signatures are the same.
 
-    # Define the control and candidate variant.
-    use { render_toggle } # control
-    try { render_button } # candidate
-  end
-end
+```ruby
+exp = Gitlab::Experiment.run(:notification_toggle, actor: user) do |e|
+  # Context may be passed in the block, but must be finalized before calling
+  # run or track.
+  e.context(project: project) # add the project to the context
 
-exp = NotificationExperiment.new(actor: user) do |e|
-  # Context may be provided within the block or to the instance (exp) but must
-  # be finalized before calling run or track.
-  e.context(project: project) # add the project id to the context
+  # Define the control and candidate variant.
+  e.use { render_toggle } # control
+  e.try { render_button } # candidate
 end
 
-# Run the experiment -- returning the result.
-exp.run
-
 # Track an event on the experiment we've defined.
 exp.track(:clicked_button)
 ```
@@ -152,11 +167,11 @@ exp.track(:clicked_button)
 </details>
 
 <details>
-  <summary>You can also specify the variant to use...</summary>
+  <summary>You can also specify the variant to use for segmentation...</summary>
 
 ### Specifying variant
 
-You can hardcode the variant if you want. It's important to know what this might do to your data during rollout, so use this with consideration.
+Generally, defining segmentation rules is a better way to approach routing into specific variants, but it's possible to explicitly specify the variant when running an experiment. It's important to know what this might do to your data during rollout, so use this with careful consideration.
 
 ```ruby
 experiment(:notification_toggle, :no_interface, actor: user) do |e|
@@ -188,6 +203,25 @@ end
 
 </details>
 
+### Segmentation rules
+
+This library comes with the capability to segment contexts into a specific variant, before asking the variant resolver which variant to provide.
+
+Segmentation can be achieved by using a custom experiment class and specifying the segmentation rules at a class level.
+
+```ruby
+class NotificationToggleExperiment < ApplicationExperiment
+  segment(variant: :variant_one) { context.actor.username == 'jejacks0n' }
+  segment(variant: :variant_two) { context.actor.created_at < 2.weeks.ago }
+end
+```
+
+In the previous examples, any user with the username `'jejacks0n'` would always receive the experience defined in "variant_one". As well, any account less than 2 weeks old would get the alternate experience defined in "variant_two".
+
+When an experiment is run, the segmentation rules are executed in the order they're defined. The first segmentation rule to produce a truthy result is the one which gets used to assign the variant. The remaining segmentation rules are skipped.
+
+This means that any user with the name `'jejacks0n'`, regardless of account age, will always be provided the experience as defined in "variant_one". 
+
 ### Return value
 
 By default the return value is a `Gitlab::Experiment` instance. In simple cases you may want only the results of the experiment though. You can call `run` within the block to get the return value of the assigned variant.
@@ -307,7 +341,7 @@ Gitlab::Experiment.configure do |config|
 end
 ```
 
-More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb).
+More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb.tt).
 
 ### Client layer / JavaScript
 
@@ -351,4 +385,32 @@ If you only include a user, that user would get the same experience across every
 
 Each of these approaches could be desirable given the objectives of your experiment.
 
-### Make code not war
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+Then, run `bundle exec rake` to run the tests. You can also run `bundle exec pry` for an
+interactive prompt that will allow you to experiment.
+
+## Contributing
+
+Bug reports and merge requests are welcome on GitLab at
+https://gitlab.com/gitlab-org/gitlab-experiment. This project is intended to be a
+safe, welcoming space for collaboration, and contributors are expected to adhere
+to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## Release Process
+
+Please refer to the [Release Process](docs/release_process.md).
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](http://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the `Gitlab::Experiment` project’s codebases, issue trackers,
+chat rooms and mailing lists is expected to follow the
+[code of conduct](CODE_OF_CONDUCT.md).
+
+***Make code not war***

data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt

@@ -7,44 +7,41 @@ Gitlab::Experiment.configure do |config|
   # The logger is used to log various details of the experiments.
   config.logger = Logger.new($stdout)
 
-  # The base class that should be instantiated for basic experiments.
+  # The base class that should be instantiated for basic experiments. It should
+  # be a string, so we can constantize it later.
   config.base_class = 'ApplicationExperiment'
 
-  # The caching layer is expected to respond to fetch, like Rails.cache.
+  # The caching layer is expected to respond to fetch, like Rails.cache for
+  # instance -- or anything that adheres to ActiveSupport::Cache::Store.
   config.cache = nil
 
+  # The domain to use on cookies.
+  #
+  # When not set, it uses the current host. If you want to provide specific
+  # hosts, you use `:all`, or provide an array like
+  # `['www.gitlab.com', '.gitlab.com']`.
+  config.cookie_domain = :all
+
   # Logic this project uses to resolve a variant for a given experiment.
   #
-  # This can return an instance of any object that responds to `name`, or can
-  # return a variant name as a string, in which case the build in variant
-  # class will be used.
+  # Should return a symbol or string that represents the variant that should
+  # be assigned. Blank or nil values will be defaulted to the control.
   #
-  # This block will be executed within the scope of the experiment instance,
-  # so can easily access experiment methods, like getting the name or context.
+  # This block is executed within the scope of the experiment and so can access
+  # experiment methods, like `name`, `context`, and `signature`.
   config.variant_resolver = lambda do |requested_variant|
     # Run the control, unless a variant was requested in code:
-    requested_variant || 'control'
+    requested_variant
 
     # Run the candidate, unless a variant was requested, with a fallback:
     #
-    # requested_variant || variant_names.first || 'control'
-
-    # Using Unleash to determine the variant:
-    #
-    # fallback = Unleash::Variant.new(name: requested_variant || 'control', enabled: true)
-    # Unleash.get_variant(name, context.value, fallback)
-
-    # Using Flipper to determine the variant:
-    #
-    # TODO: provide example.
-    # Variant.new(name: requested_variant || 'control')
+    # requested_variant || variant_names.first || nil
   end
 
   # Tracking behavior can be implemented to link an event to an experiment.
   #
-  # Similar to the variant_resolver, this is called within the scope of the
-  # experiment instance and so can access any methods on the experiment,
-  # such as name and signature.
+  # This block is executed within the scope of the experiment and so can access
+  # experiment methods, like `name`, `context`, and `signature`.
   config.tracking_behavior = lambda do |event, args|
     # An example of using a generic logger to track events:
     config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
@@ -61,8 +58,11 @@ Gitlab::Experiment.configure do |config|
   # Called at the end of every experiment run, with the result.
   #
   # You may want to track that you've assigned a variant to a given context,
-  # or push the experiment into the client or publish results elsewhere, like
-  # into redis. Also called within the scope of the experiment instance.
+  # or push the experiment into the client or publish results elsewhere like
+  # into redis.
+  #
+  # This block is executed within the scope of the experiment and so can access
+  # experiment methods, like `name`, `context`, and `signature`.
   config.publishing_behavior = lambda do |result|
     # Track the event using our own configured tracking logic.
     track(:assignment)
@@ -83,14 +83,11 @@ Gitlab::Experiment.configure do |config|
   # Given a specific context hash map, we need to generate a consistent hash
   # key. The logic in here will be used for generating cache keys, and may also
   # be used when determining which variant may be presented.
+  #
+  # This block is executed within the scope of the experiment and so can access
+  # experiment methods, like `name`, `context`, and `signature`.
   config.context_hash_strategy = lambda do |context|
     values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
     Digest::MD5.hexdigest((context.keys + values).join('|'))
   end
-
-  # The domain for which this cookie applies so you can restrict to the domain level.
-  #
-  # When not set, it uses the current host. If you want to provide specific hosts, you can
-  # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
-  config.cookie_domain = :all
 end

data/lib/gitlab/experiment.rb

@@ -30,7 +30,7 @@ module Gitlab
         raise ArgumentError, 'name is required' if name.nil? && base?
 
         instance = constantize(name).new(name, variant_name, **context, &block)
-        return instance unless block_given?
+        return instance unless block
 
         instance.context.frozen? ? instance.run : instance.tap(&:run)
       end
@@ -58,9 +58,9 @@ module Gitlab
       raise ArgumentError, 'name is required' if name.blank? && self.class.base?
 
       @name = self.class.experiment_name(name, suffix: false)
-      @variant_name = variant_name
       @excluded = []
-      @context = Context.new(self, context)
+      @context = Context.new(self, **context)
+      @variant_name = cache_variant(variant_name) { nil } if variant_name.present?
 
       exclude { !@context.trackable? }
       compare { false }
@@ -76,10 +76,22 @@ module Gitlab
     end
 
     def variant(value = nil)
-      return @variant_name = value unless value.nil?
+      if value.blank? && @variant_name || @resolving_variant
+        return Variant.new(name: (@variant_name || :unresolved).to_s)
+      end
+
+      @variant_name = value unless value.blank?
+      @variant_name ||= :control if excluded?
+
+      @resolving_variant = true
+      resolved = :control
+      if (result = cache_variant(@variant_name) { resolve_variant_name }).present?
+        @variant_name = resolved = result.to_sym
+      end
 
-      result = instance_exec(@variant_name, &Configuration.variant_resolver)
-      result.respond_to?(:name) ? result : Variant.new(name: result.to_s)
+      Variant.new(name: resolved.to_s)
+    ensure
+      @resolving_variant = false
     end
 
     def exclude(&block)
@@ -88,19 +100,9 @@ module Gitlab
 
     def run(variant_name = nil)
       @result ||= begin
-        @variant_name = variant_name unless variant_name.nil?
-        @variant_name ||= :control if excluded?
-
-        chain = variant_assigned? ? :unsegmented_run : :segmented_run
-        run_callbacks(chain) do
-          variant_name = cache { variant.name }
-
-          method_name = "#{variant_name}_behavior"
-          if respond_to?(method_name)
-            behaviors[variant_name] ||= -> { send(method_name) } # rubocop:disable GitlabSecurity/PublicSend
-          end
-
-          super(variant_name)
+        variant_name = variant(variant_name).name
+        run_callbacks(variant_assigned? ? :unsegmented_run : :segmented_run) do
+          super(@variant_name ||= variant_name)
         end
       end
     end
@@ -123,6 +125,20 @@ module Gitlab
       @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
     end
 
+    def behaviors
+      @behaviors ||= public_methods.each_with_object(super) do |name, behaviors|
+        next unless name.end_with?('_behavior')
+
+        behavior_name = name.to_s.sub(/_behavior$/, '')
+        behaviors[behavior_name] ||= -> { send(name) } # rubocop:disable GitlabSecurity/PublicSend
+      end
+    end
+
+    def try(name = nil, &block)
+      name = (name || 'candidate').to_s
+      behaviors[name] = block
+    end
+
     def signature
       { variant: variant.name, experiment: name }.merge(context.signature)
     end
@@ -140,7 +156,7 @@ module Gitlab
     end
 
     def id
-      "#{name}:#{signature[:key]}"
+      "#{name}:#{key_for(context.value)}"
     end
     alias_method :session_id, :id
 
@@ -154,6 +170,10 @@ module Gitlab
 
     protected
 
+    def resolve_variant_name
+      instance_exec(@variant_name, &Configuration.variant_resolver)
+    end
+
     def generate_result(variant_name)
       observation = Scientist::Observation.new(variant_name, self, &behaviors[variant_name])
       Scientist::Result.new(self, [observation], observation)

data/lib/gitlab/experiment/caching.rb

@@ -3,20 +3,26 @@
 module Gitlab
   class Experiment
     module Caching
-      def cache(&block)
-        return yield unless (cache = Configuration.cache)
+      def cache_variant(specified = nil, &block)
+        cache = Configuration.cache
+        return (specified.presence || yield) unless cache
 
         key, migrations = cache_strategy
-        migrated_cache(cache, migrations || [], key) or cache.fetch(key, &block)
+        result = migrated_cache(cache, migrations || [], key) || cache.fetch(key, &block)
+        return result unless specified.present?
+
+        cache.write(cache_key, specified) if result != specified
+        specified
+      end
+
+      def cache_key(key = nil)
+        "#{name}:#{key || context.signature[:key]}"
       end
 
       private
 
       def cache_strategy
-        [
-          "#{name}:#{signature[:key]}",
-          signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
-        ]
+        [cache_key, context.signature[:migration_keys]&.map { |key| cache_key(key) }]
       end
 
       def migrated_cache(cache, migrations, new_key)

data/lib/gitlab/experiment/callbacks.rb

@@ -7,31 +7,20 @@ module Gitlab
       include ActiveSupport::Callbacks
 
       included do
-        define_callbacks(
-          :unsegmented_run,
-          skip_after_callbacks_if_terminated: true
-        )
-
-        define_callbacks(
-          :segmented_run,
-          skip_after_callbacks_if_terminated: false,
-          terminator: lambda do |target, result_lambda|
-            result_lambda.call
-            target.variant_assigned?
-          end
-        )
+        define_callbacks(:unsegmented_run)
+        define_callbacks(:segmented_run)
       end
 
       class_methods do
         def segment(*filter_list, variant:, **options, &block)
           filters = filter_list.unshift(block).compact.map do |filter|
             result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
-            ->(target) { target.variant(variant) if result_lambda.call(target, nil) }
+            ->(target) { target.variant(variant) if !target.variant_assigned? && result_lambda.call(target, nil) }
           end
 
           raise ArgumentError, 'no filters provided' if filters.empty?
 
-          set_callback(:segmented_run, :before, *filters, options, &block)
+          set_callback(:segmented_run, :before, *filters, options)
         end
       end
     end

data/lib/gitlab/experiment/configuration.rb

@@ -18,12 +18,16 @@ module Gitlab
       # The base class that should be instantiated for basic experiments.
       @base_class = 'Gitlab::Experiment'
 
-      # Cache layer. Expected to respond to fetch, like Rails.cache.
+      # The caching layer is expected to respond to fetch, like Rails.cache.
       @cache = nil
 
+      # The domain to use on cookies.
+      @cookie_domain = :all
+
       # Logic this project uses to resolve a variant for a given experiment.
+      # If no variant is determined, the control will be used.
       @variant_resolver = lambda do |requested_variant|
-        requested_variant || 'control'
+        requested_variant
       end
 
       # Tracking behavior can be implemented to link an event to an experiment.
@@ -31,8 +35,7 @@ module Gitlab
         Configuration.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
       end
 
-      # Called at the end of every experiment run, with the results. You may
-      # want to push the experiment into the client or push results elsewhere.
+      # Called at the end of every experiment run, with the result.
       @publishing_behavior = lambda do |_result|
         track(:assignment)
       end
@@ -43,22 +46,17 @@ module Gitlab
         Digest::MD5.hexdigest(([name] + hash_map.keys + values).join('|'))
       end
 
-      # The domain for which this cookie applies so you can restrict to the domain level.
-      # When not set, it uses the current host. If you want to provide specific hosts, you can
-      # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
-      @cookie_domain = :all
-
       class << self
         attr_accessor(
           :name_prefix,
           :logger,
           :base_class,
           :cache,
+          :cookie_domain,
           :variant_resolver,
           :tracking_behavior,
           :publishing_behavior,
-          :context_hash_strategy,
-          :cookie_domain
+          :context_hash_strategy
         )
       end
     end

data/lib/gitlab/experiment/cookies.rb

@@ -11,7 +11,7 @@ module Gitlab
         return hash if cookie_jar.nil?
 
         resolver = [hash, :actor, cookie_name, cookie_jar.signed[cookie_name]]
-        resolve_cookie(*resolver) or generate_cookie(*resolver)
+        resolve_cookie(*resolver) || generate_cookie(*resolver)
       end
 
       def cookie_jar

data/lib/gitlab/experiment/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   class Experiment
-    VERSION = '0.4.0'
+    VERSION = '0.4.5'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-experiment
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.5
 platform: ruby
 authors:
 - GitLab
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-17 00:00:00.000000000 Z
+date: 2021-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport