gitlab-experiment 0.2.2 → 0.4.0

data/lib/generators/gitlab/experiment/experiment_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Gitlab
+  module Generators
+    class ExperimentGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates/', __dir__)
+      check_class_collision suffix: 'Experiment'
+
+      argument :variants,
+               type: :array,
+               default: %w[control candidate],
+               banner: 'variant variant'
+
+      def create_experiment
+        template 'experiment.rb', File.join('app/experiments', class_path, "#{file_name}_experiment.rb")
+      end
+
+      hook_for :test_framework
+
+      private
+
+      def file_name
+        @_file_name ||= remove_possible_suffix(super)
+      end
+
+      def remove_possible_suffix(name)
+        name.sub(/_?exp[ei]riment$/i, "") # be somewhat forgiving with spelling
+      end
+    end
+  end
+end

data/lib/generators/gitlab/experiment/install/install_generator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Gitlab
+  module Generators
+    module Experiment
+      class InstallGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        desc 'Installs the Gitlab::Experiment initializer and optional ApplicationExperiment into your application.'
+
+        class_option :skip_initializer,
+                     type: :boolean,
+                     default: false,
+                     desc: 'Skip the initializer with default configuration'
+
+        class_option :skip_baseclass,
+                     type: :boolean,
+                     default: false,
+                     desc: 'Skip the ApplicationExperiment base class'
+
+        def create_initializer
+          return if options[:skip_initializer]
+
+          template 'initializer.rb', 'config/initializers/gitlab_experiment.rb'
+        end
+
+        def create_baseclass
+          return if options[:skip_baseclass]
+
+          template 'application_experiment.rb', 'app/experiments/application_experiment.rb'
+        end
+
+        def display_post_install
+          readme 'POST_INSTALL' if behavior == :invoke
+        end
+      end
+    end
+  end
+end

data/lib/generators/gitlab/experiment/install/templates/POST_INSTALL

@@ -0,0 +1,2 @@
+Gitlab::Experiment has been installed. You may want to adjust the configuration
+that's been provided in the Rails initializer.

data/lib/generators/gitlab/experiment/install/templates/application_experiment.rb.tt

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ApplicationExperiment < Gitlab::Experiment
+end

data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+Gitlab::Experiment.configure do |config|
+  # Prefix all experiment names with a given value. Use `nil` for none.
+  config.name_prefix = nil
+
+  # The logger is used to log various details of the experiments.
+  config.logger = Logger.new($stdout)
+
+  # The base class that should be instantiated for basic experiments.
+  config.base_class = 'ApplicationExperiment'
+
+  # The caching layer is expected to respond to fetch, like Rails.cache.
+  config.cache = nil
+
+  # Logic this project uses to resolve a variant for a given experiment.
+  #
+  # This can return an instance of any object that responds to `name`, or can
+  # return a variant name as a string, in which case the build in variant
+  # class will be used.
+  #
+  # This block will be executed within the scope of the experiment instance,
+  # so can easily access experiment methods, like getting the name or context.
+  config.variant_resolver = lambda do |requested_variant|
+    # Run the control, unless a variant was requested in code:
+    requested_variant || 'control'
+
+    # Run the candidate, unless a variant was requested, with a fallback:
+    #
+    # requested_variant || variant_names.first || 'control'
+
+    # Using Unleash to determine the variant:
+    #
+    # fallback = Unleash::Variant.new(name: requested_variant || 'control', enabled: true)
+    # Unleash.get_variant(name, context.value, fallback)
+
+    # Using Flipper to determine the variant:
+    #
+    # TODO: provide example.
+    # Variant.new(name: requested_variant || 'control')
+  end
+
+  # Tracking behavior can be implemented to link an event to an experiment.
+  #
+  # Similar to the variant_resolver, this is called within the scope of the
+  # experiment instance and so can access any methods on the experiment,
+  # such as name and signature.
+  config.tracking_behavior = lambda do |event, args|
+    # An example of using a generic logger to track events:
+    config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
+
+    # Using something like snowplow to track events (in gitlab):
+    #
+    # Gitlab::Tracking.event(name, event, **args.merge(
+    #   context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
+    #     'iglu:com.gitlab/gitlab_experiment/jsonschema/0-2-0', signature
+    #   )
+    # ))
+  end
+
+  # Called at the end of every experiment run, with the result.
+  #
+  # You may want to track that you've assigned a variant to a given context,
+  # or push the experiment into the client or publish results elsewhere, like
+  # into redis. Also called within the scope of the experiment instance.
+  config.publishing_behavior = lambda do |result|
+    # Track the event using our own configured tracking logic.
+    track(:assignment)
+
+    # Push the experiment knowledge into the front end. The signature contains
+    # the context key, and the variant that has been determined.
+    #
+    # Gon.push({ experiment: { name => signature } }, true)
+
+    # Log using our logging system, so the result (which can be large) can be
+    # reviewed later if we want to.
+    #
+    # Lograge::Event.log(experiment: name, result: result, signature: signature)
+  end
+
+  # Algorithm that consistently generates a hash key for a given hash map.
+  #
+  # Given a specific context hash map, we need to generate a consistent hash
+  # key. The logic in here will be used for generating cache keys, and may also
+  # be used when determining which variant may be presented.
+  config.context_hash_strategy = lambda do |context|
+    values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
+    Digest::MD5.hexdigest((context.keys + values).join('|'))
+  end
+
+  # The domain for which this cookie applies so you can restrict to the domain level.
+  #
+  # When not set, it uses the current host. If you want to provide specific hosts, you can
+  # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
+  config.cookie_domain = :all
+end

data/lib/generators/gitlab/experiment/templates/experiment.rb.tt

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+<% if namespaced? -%>
+require_dependency "<%= namespaced_path %>/application_experiment"
+
+<% end -%>
+<% module_namespacing do -%>
+class <%= class_name %>Experiment < ApplicationExperiment
+<% variants.each do |variant| -%>
+  def <%= variant %>_behavior
+  end
+<%= "\n" unless variant == variants.last -%>
+<% end -%>
+end
+<% end -%>

data/lib/generators/rspec/experiment/experiment_generator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'generators/rspec'
+
+module Rspec
+  module Generators
+    class ExperimentGenerator < Rspec::Generators::Base
+      source_root File.expand_path('templates/', __dir__)
+
+      def create_experiment_spec
+        template 'experiment_spec.rb', File.join('spec/experiments', class_path, "#{file_name}_experiment_spec.rb")
+      end
+    end
+  end
+end

data/lib/generators/rspec/experiment/templates/experiment_spec.rb.tt

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Experiment do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/test_unit/experiment/experiment_generator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rails/generators/test_unit'
+
+module TestUnit # :nodoc:
+  module Generators # :nodoc:
+    class ExperimentGenerator < TestUnit::Generators::Base # :nodoc:
+      source_root File.expand_path('templates/', __dir__)
+
+      check_class_collision suffix: 'Test'
+
+      def create_test_file
+        template 'experiment_test.rb', File.join('test/experiments', class_path, "#{file_name}_experiment_test.rb")
+      end
+    end
+  end
+end

data/lib/generators/test_unit/experiment/templates/experiment_test.rb.tt

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+<% module_namespacing do -%>
+class <%= class_name %>ExperimentTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end
+<% end -%>

data/lib/gitlab/experiment.rb

@@ -1,8 +1,14 @@
 # frozen_string_literal: true
 
 require 'scientist'
+require 'active_support/callbacks'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/string/inflections'
 
+require 'gitlab/experiment/caching'
+require 'gitlab/experiment/callbacks'
 require 'gitlab/experiment/configuration'
+require 'gitlab/experiment/cookies'
 require 'gitlab/experiment/context'
 require 'gitlab/experiment/dsl'
 require 'gitlab/experiment/variant'
@@ -12,50 +18,91 @@ require 'gitlab/experiment/engine' if defined?(Rails::Engine)
 module Gitlab
   class Experiment
     include Scientist::Experiment
+    include Caching
+    include Callbacks
 
     class << self
       def configure
         yield Configuration
       end
 
-      def run(name, variant_name = nil, **context, &block)
-        instance = new(name, variant_name, **context, &block)
+      def run(name = nil, variant_name = nil, **context, &block)
+        raise ArgumentError, 'name is required' if name.nil? && base?
+
+        instance = constantize(name).new(name, variant_name, **context, &block)
         return instance unless block_given?
 
         instance.context.frozen? ? instance.run : instance.tap(&:run)
       end
+
+      def experiment_name(name = nil, suffix: true, suffix_word: 'experiment')
+        name = (name.presence || self.name).to_s.underscore.sub(%r{(?<char>[_/]|)#{suffix_word}$}, '')
+        name = "#{name}#{Regexp.last_match(:char) || '_'}#{suffix_word}"
+        suffix ? name : name.sub(/_#{suffix_word}$/, '')
+      end
+
+      def base?
+        self == Gitlab::Experiment || name == Configuration.base_class
+      end
+
+      private
+
+      def constantize(name = nil)
+        return self if name.nil?
+
+        experiment_name(name).classify.safe_constantize || Configuration.base_class.constantize
+      end
     end
 
-    delegate :signature, to: :context
+    def initialize(name = nil, variant_name = nil, **context)
+      raise ArgumentError, 'name is required' if name.blank? && self.class.base?
 
-    def initialize(name, variant_name = nil, **context)
-      @name = name
+      @name = self.class.experiment_name(name, suffix: false)
       @variant_name = variant_name
-      @context = Context.new(self)
-
-      context(context)
+      @excluded = []
+      @context = Context.new(self, context)
 
-      ignore { true }
+      exclude { !@context.trackable? }
       compare { false }
 
       yield self if block_given?
     end
 
     def context(value = nil)
-      return @context if value.nil?
+      return @context if value.blank?
 
       @context.value(value)
       @context
     end
 
     def variant(value = nil)
-      @variant_name = value unless value.nil?
+      return @variant_name = value unless value.nil?
+
       result = instance_exec(@variant_name, &Configuration.variant_resolver)
       result.respond_to?(:name) ? result : Variant.new(name: result.to_s)
     end
 
-    def run
-      @result ||= super(variant.name)
+    def exclude(&block)
+      @excluded << block
+    end
+
+    def run(variant_name = nil)
+      @result ||= begin
+        @variant_name = variant_name unless variant_name.nil?
+        @variant_name ||= :control if excluded?
+
+        chain = variant_assigned? ? :unsegmented_run : :segmented_run
+        run_callbacks(chain) do
+          variant_name = cache { variant.name }
+
+          method_name = "#{variant_name}_behavior"
+          if respond_to?(method_name)
+            behaviors[variant_name] ||= -> { send(method_name) } # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          super(variant_name)
+        end
+      end
     end
 
     def publish(result)
@@ -63,6 +110,8 @@ module Gitlab
     end
 
     def track(action, **event_args)
+      return if excluded?
+
       instance_exec(action, event_args, &Configuration.tracking_behavior)
     end
 
@@ -71,13 +120,38 @@ module Gitlab
     end
 
     def variant_names
-      @variant_names = behaviors.keys.tap { |keys| keys.delete('control') }.map(&:to_sym)
+      @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
+    end
+
+    def signature
+      { variant: variant.name, experiment: name }.merge(context.signature)
     end
 
     def enabled?
       true
     end
 
+    def excluded?
+      @excluded.any? { |exclude| exclude.call(self) }
+    end
+
+    def variant_assigned?
+      !@variant_name.nil?
+    end
+
+    def id
+      "#{name}:#{signature[:key]}"
+    end
+    alias_method :session_id, :id
+
+    def flipper_id
+      "Experiment;#{id}"
+    end
+
+    def key_for(hash)
+      instance_exec(hash, &Configuration.context_hash_strategy)
+    end
+
     protected
 
     def generate_result(variant_name)

data/lib/gitlab/experiment/caching.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Experiment
+    module Caching
+      def cache(&block)
+        return yield unless (cache = Configuration.cache)
+
+        key, migrations = cache_strategy
+        migrated_cache(cache, migrations || [], key) or cache.fetch(key, &block)
+      end
+
+      private
+
+      def cache_strategy
+        [
+          "#{name}:#{signature[:key]}",
+          signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
+        ]
+      end
+
+      def migrated_cache(cache, migrations, new_key)
+        migrations.find do |old_key|
+          next unless (value = cache.read(old_key))
+
+          cache.write(new_key, value)
+          cache.delete(old_key)
+          break value
+        end
+      end
+    end
+  end
+end