gitlab-experiment 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4bb028b4495b7ab210c13769433a8b6ec7b8a3fc459a2c33f0e9bc650cf23916
-  data.tar.gz: '0387ed9d7e722c4ba6bce42aed1b71720b6df23b6687100b02186f57b6f125c9'
+  metadata.gz: 8524dff7f908e481923e3131f3dd67c34f426e31ed10171896ea10a209e0b74a
+  data.tar.gz: 151f2e7f7bbf9692350c02f46f0c4c8cfa6f9c7ec05fdcb0fda50e34324c06fb
 SHA512:
-  metadata.gz: d27ae9c243b447d994060bc2250ade8be78351312c01fa591e67441f913fd67f9e2caed1de77caac454b0a6b84702ff865e6a58c796c8e014ba2cf530dd5df9b
-  data.tar.gz: 2a3dcfef0dbfeb7e9e35d3ca56ede0bc7d1feeeafa9774a0f9184eeb89dfe49a49e6fbc93f594332cda1d8c87b374069a0ab1c053ab51183d8fdf47af4769845
+  metadata.gz: 3f23698aabf3968c77f49cdb924125879663670c12d22c390a47f7c791687f87fb1211adf432777f66012e3be1db2e90d5c33a1dc0a156332bb8926938c4e587
+  data.tar.gz: 51973494136edef02672c9086970fec2ecb1fb3a6716edb10d1cfe758172f8d256ecd6ef1b5cefdd50133898637e022af527fc7e20df2212b1f7210db2dd087e

data/README.md

@@ -4,7 +4,7 @@
 
 Here at GitLab, we run experiments as A/B/n tests and review the data the experiment generates. From that data, we determine the best performing variant and promote it as the new default code path. Or revert back to the control if no variant outperformed it.
 
-This library provides a clean and elegant DSL to define, run, and track your GitLab experiment.
+This library provides a clean and elegant DSL (domain specific language) to define, run, and track your GitLab experiment.
 
 When we discuss the behavior of this gem, we'll use terms like experiment, context, control, candidate, and variant. It's worth defining these terms so they're more understood.
 
@@ -27,7 +27,7 @@ gem 'gitlab-experiment'
 If you're using Rails, you can install the initializer. It provides basic configuration and documentation.
 
 ```shell
-$ rails generate gitlab_experiment:install
+$ rails generate gitlab:experiment:install
 ```
 
 ## Implementing an experiment
@@ -307,7 +307,7 @@ Gitlab::Experiment.configure do |config|
 end
 ```
 
-More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab_experiment/install/templates/initializer.rb).
+More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb).
 
 ### Client layer / JavaScript
 

data/lib/generators/gitlab/experiment/USAGE

@@ -0,0 +1,17 @@
+Description:
+    Stubs out a new experiment and its variants. Pass the experiment name,
+    either CamelCased or under_scored, and a list of variants as arguments.
+
+    To create an experiment within a module, specify the experiment name as a
+    path like 'parent_module/experiment_name'.
+
+    This generates an experiment class in app/experiments and invokes feature
+    flag, and test framework generators.
+
+Example:
+    `rails generate gitlab:experiment NullHypothesis control candidate alt_variant`
+
+    NullHypothesis experiment with default variants.
+        Experiment:   app/experiments/null_hypothesis_experiment.rb
+        Feature Flag: config/feature_flags/experiment/null_hypothesis.yaml
+        Test:         test/experiments/null_hypothesis_experiment_test.rb

data/lib/generators/gitlab/experiment/experiment_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Gitlab
+  module Generators
+    class ExperimentGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates/', __dir__)
+      check_class_collision suffix: 'Experiment'
+
+      argument :variants,
+               type: :array,
+               default: %w[control candidate],
+               banner: 'variant variant'
+
+      def create_experiment
+        template 'experiment.rb', File.join('app/experiments', class_path, "#{file_name}_experiment.rb")
+      end
+
+      hook_for :test_framework
+
+      private
+
+      def file_name
+        @_file_name ||= remove_possible_suffix(super)
+      end
+
+      def remove_possible_suffix(name)
+        name.sub(/_?exp[ei]riment$/i, "") # be somewhat forgiving with spelling
+      end
+    end
+  end
+end

data/lib/generators/gitlab/experiment/install/install_generator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Gitlab
+  module Generators
+    module Experiment
+      class InstallGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        desc 'Installs the Gitlab::Experiment initializer and optional ApplicationExperiment into your application.'
+
+        class_option :skip_initializer,
+                     type: :boolean,
+                     default: false,
+                     desc: 'Skip the initializer with default configuration'
+
+        class_option :skip_baseclass,
+                     type: :boolean,
+                     default: false,
+                     desc: 'Skip the ApplicationExperiment base class'
+
+        def create_initializer
+          return if options[:skip_initializer]
+
+          template 'initializer.rb', 'config/initializers/gitlab_experiment.rb'
+        end
+
+        def create_baseclass
+          return if options[:skip_baseclass]
+
+          template 'application_experiment.rb', 'app/experiments/application_experiment.rb'
+        end
+
+        def display_post_install
+          readme 'POST_INSTALL' if behavior == :invoke
+        end
+      end
+    end
+  end
+end

data/lib/generators/gitlab/experiment/install/templates/POST_INSTALL

@@ -0,0 +1,2 @@
+Gitlab::Experiment has been installed. You may want to adjust the configuration
+that's been provided in the Rails initializer.

data/lib/generators/gitlab/experiment/install/templates/application_experiment.rb.tt

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ApplicationExperiment < Gitlab::Experiment
+end

data/lib/generators/{gitlab_experiment/install/templates/initializer.rb → gitlab/experiment/install/templates/initializer.rb.tt}

@@ -7,6 +7,9 @@ Gitlab::Experiment.configure do |config|
   # The logger is used to log various details of the experiments.
   config.logger = Logger.new($stdout)
 
+  # The base class that should be instantiated for basic experiments.
+  config.base_class = 'ApplicationExperiment'
+
   # The caching layer is expected to respond to fetch, like Rails.cache.
   config.cache = nil
 
@@ -84,4 +87,10 @@ Gitlab::Experiment.configure do |config|
     values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
     Digest::MD5.hexdigest((context.keys + values).join('|'))
   end
+
+  # The domain for which this cookie applies so you can restrict to the domain level.
+  #
+  # When not set, it uses the current host. If you want to provide specific hosts, you can
+  # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
+  config.cookie_domain = :all
 end

data/lib/generators/gitlab/experiment/templates/experiment.rb.tt

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+<% if namespaced? -%>
+require_dependency "<%= namespaced_path %>/application_experiment"
+
+<% end -%>
+<% module_namespacing do -%>
+class <%= class_name %>Experiment < ApplicationExperiment
+<% variants.each do |variant| -%>
+  def <%= variant %>_behavior
+  end
+<%= "\n" unless variant == variants.last -%>
+<% end -%>
+end
+<% end -%>

data/lib/generators/rspec/experiment/experiment_generator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'generators/rspec'
+
+module Rspec
+  module Generators
+    class ExperimentGenerator < Rspec::Generators::Base
+      source_root File.expand_path('templates/', __dir__)
+
+      def create_experiment_spec
+        template 'experiment_spec.rb', File.join('spec/experiments', class_path, "#{file_name}_experiment_spec.rb")
+      end
+    end
+  end
+end

data/lib/generators/rspec/experiment/templates/experiment_spec.rb.tt

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Experiment do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/test_unit/experiment/experiment_generator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rails/generators/test_unit'
+
+module TestUnit # :nodoc:
+  module Generators # :nodoc:
+    class ExperimentGenerator < TestUnit::Generators::Base # :nodoc:
+      source_root File.expand_path('templates/', __dir__)
+
+      check_class_collision suffix: 'Test'
+
+      def create_test_file
+        template 'experiment_test.rb', File.join('test/experiments', class_path, "#{file_name}_experiment_test.rb")
+      end
+    end
+  end
+end

data/lib/generators/test_unit/experiment/templates/experiment_test.rb.tt

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+<% module_namespacing do -%>
+class <%= class_name %>ExperimentTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end
+<% end -%>

data/lib/gitlab/experiment.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
 require 'scientist'
+require 'active_support/callbacks'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/string/inflections'
 
 require 'gitlab/experiment/caching'
+require 'gitlab/experiment/callbacks'
 require 'gitlab/experiment/configuration'
 require 'gitlab/experiment/cookies'
 require 'gitlab/experiment/context'
@@ -15,28 +19,45 @@ module Gitlab
   class Experiment
     include Scientist::Experiment
     include Caching
+    include Callbacks
 
     class << self
       def configure
         yield Configuration
       end
 
-      def run(name, variant_name = nil, **context, &block)
+      def run(name = nil, variant_name = nil, **context, &block)
+        raise ArgumentError, 'name is required' if name.nil? && base?
+
         instance = constantize(name).new(name, variant_name, **context, &block)
         return instance unless block_given?
 
         instance.context.frozen? ? instance.run : instance.tap(&:run)
       end
 
-      def constantize(name)
-        name = "#{name}_experiment"
-        klass = name.respond_to?(:classify) ? name.classify.safe_constantize : nil
-        klass || self
+      def experiment_name(name = nil, suffix: true, suffix_word: 'experiment')
+        name = (name.presence || self.name).to_s.underscore.sub(%r{(?<char>[_/]|)#{suffix_word}$}, '')
+        name = "#{name}#{Regexp.last_match(:char) || '_'}#{suffix_word}"
+        suffix ? name : name.sub(/_#{suffix_word}$/, '')
+      end
+
+      def base?
+        self == Gitlab::Experiment || name == Configuration.base_class
+      end
+
+      private
+
+      def constantize(name = nil)
+        return self if name.nil?
+
+        experiment_name(name).classify.safe_constantize || Configuration.base_class.constantize
       end
     end
 
-    def initialize(name, variant_name = nil, **context)
-      @name = name
+    def initialize(name = nil, variant_name = nil, **context)
+      raise ArgumentError, 'name is required' if name.blank? && self.class.base?
+
+      @name = self.class.experiment_name(name, suffix: false)
       @variant_name = variant_name
       @excluded = []
       @context = Context.new(self, context)
@@ -48,7 +69,7 @@ module Gitlab
     end
 
     def context(value = nil)
-      return @context if value.nil?
+      return @context if value.blank?
 
       @context.value(value)
       @context
@@ -70,7 +91,17 @@ module Gitlab
         @variant_name = variant_name unless variant_name.nil?
         @variant_name ||= :control if excluded?
 
-        super(cache { variant.name })
+        chain = variant_assigned? ? :unsegmented_run : :segmented_run
+        run_callbacks(chain) do
+          variant_name = cache { variant.name }
+
+          method_name = "#{variant_name}_behavior"
+          if respond_to?(method_name)
+            behaviors[variant_name] ||= -> { send(method_name) } # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          super(variant_name)
+        end
       end
     end
 
@@ -104,6 +135,10 @@ module Gitlab
       @excluded.any? { |exclude| exclude.call(self) }
     end
 
+    def variant_assigned?
+      !@variant_name.nil?
+    end
+
     def id
       "#{name}:#{signature[:key]}"
     end
@@ -113,6 +148,10 @@ module Gitlab
       "Experiment;#{id}"
     end
 
+    def key_for(hash)
+      instance_exec(hash, &Configuration.context_hash_strategy)
+    end
+
     protected
 
     def generate_result(variant_name)

data/lib/gitlab/experiment/callbacks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class Experiment
+    module Callbacks
+      extend ActiveSupport::Concern
+      include ActiveSupport::Callbacks
+
+      included do
+        define_callbacks(
+          :unsegmented_run,
+          skip_after_callbacks_if_terminated: true
+        )
+
+        define_callbacks(
+          :segmented_run,
+          skip_after_callbacks_if_terminated: false,
+          terminator: lambda do |target, result_lambda|
+            result_lambda.call
+            target.variant_assigned?
+          end
+        )
+      end
+
+      class_methods do
+        def segment(*filter_list, variant:, **options, &block)
+          filters = filter_list.unshift(block).compact.map do |filter|
+            result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
+            ->(target) { target.variant(variant) if result_lambda.call(target, nil) }
+          end
+
+          raise ArgumentError, 'no filters provided' if filters.empty?
+
+          set_callback(:segmented_run, :before, *filters, options, &block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/experiment/configuration.rb

@@ -15,6 +15,9 @@ module Gitlab
       # The logger is used to log various details of the experiments.
       @logger = Logger.new($stdout)
 
+      # The base class that should be instantiated for basic experiments.
+      @base_class = 'Gitlab::Experiment'
+
       # Cache layer. Expected to respond to fetch, like Rails.cache.
       @cache = nil
 
@@ -35,20 +38,27 @@ module Gitlab
       end
 
       # Algorithm that consistently generates a hash key for a given hash map.
-      @context_hash_strategy = lambda do |context|
-        values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
-        Digest::MD5.hexdigest((context.keys + values).join('|'))
+      @context_hash_strategy = lambda do |hash_map|
+        values = hash_map.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
+        Digest::MD5.hexdigest(([name] + hash_map.keys + values).join('|'))
       end
 
+      # The domain for which this cookie applies so you can restrict to the domain level.
+      # When not set, it uses the current host. If you want to provide specific hosts, you can
+      # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
+      @cookie_domain = :all
+
       class << self
         attr_accessor(
           :name_prefix,
           :logger,
+          :base_class,
           :cache,
           :variant_resolver,
           :tracking_behavior,
           :publishing_behavior,
-          :context_hash_strategy
+          :context_hash_strategy,
+          :cookie_domain
         )
       end
     end

data/lib/gitlab/experiment/context.rb

@@ -39,7 +39,7 @@ module Gitlab
       end
 
       def signature
-        @signature ||= { key: key_for(@value), migration_keys: migration_keys }.compact
+        @signature ||= { key: @experiment.key_for(@value), migration_keys: migration_keys }.compact
       end
 
       private
@@ -60,12 +60,8 @@ module Gitlab
       def migration_keys
         return nil if @migrations[:unmerged].empty? && @migrations[:merged].empty?
 
-        @migrations[:unmerged].map { |m| key_for(m) } +
-          @migrations[:merged].map { |m| key_for(@value.merge(m)) }
-      end
-
-      def key_for(context)
-        Configuration.context_hash_strategy.call(context)
+        @migrations[:unmerged].map { |m| @experiment.key_for(m) } +
+          @migrations[:merged].map { |m| @experiment.key_for(@value.merge(m)) }
       end
     end
   end

data/lib/gitlab/experiment/cookies.rb

@@ -24,7 +24,7 @@ module Gitlab
         return hash.merge(key => cookie) if hash[key].nil?
 
         add_migration(key => cookie)
-        cookie_jar.delete(cookie_name, domain: :all)
+        cookie_jar.delete(cookie_name, domain: domain)
 
         hash
       end
@@ -34,11 +34,15 @@ module Gitlab
 
         cookie ||= SecureRandom.uuid
         cookie_jar.permanent.signed[cookie_name] = {
-          value: cookie, secure: true, domain: :all, httponly: true
+          value: cookie, secure: true, domain: domain, httponly: true
         }
 
         hash.merge(key => cookie)
       end
+
+      def domain
+        Configuration.cookie_domain
+      end
     end
   end
 end

data/lib/gitlab/experiment/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   class Experiment
-    VERSION = '0.3.1'
+    VERSION = '0.4.0'
   end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-experiment
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - GitLab
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2020-11-17 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: scientist
   requirement: !ruby/object:Gem::Requirement
@@ -39,11 +53,20 @@ extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
-- lib/generators/gitlab_experiment/install/POST_INSTALL
-- lib/generators/gitlab_experiment/install/install_generator.rb
-- lib/generators/gitlab_experiment/install/templates/initializer.rb
+- lib/generators/gitlab/experiment/USAGE
+- lib/generators/gitlab/experiment/experiment_generator.rb
+- lib/generators/gitlab/experiment/install/install_generator.rb
+- lib/generators/gitlab/experiment/install/templates/POST_INSTALL
+- lib/generators/gitlab/experiment/install/templates/application_experiment.rb.tt
+- lib/generators/gitlab/experiment/install/templates/initializer.rb.tt
+- lib/generators/gitlab/experiment/templates/experiment.rb.tt
+- lib/generators/rspec/experiment/experiment_generator.rb
+- lib/generators/rspec/experiment/templates/experiment_spec.rb.tt
+- lib/generators/test_unit/experiment/experiment_generator.rb
+- lib/generators/test_unit/experiment/templates/experiment_test.rb.tt
 - lib/gitlab/experiment.rb
 - lib/gitlab/experiment/caching.rb
+- lib/gitlab/experiment/callbacks.rb
 - lib/gitlab/experiment/configuration.rb
 - lib/gitlab/experiment/context.rb
 - lib/gitlab/experiment/cookies.rb

data/lib/generators/gitlab_experiment/install/install_generator.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails/generators'
-
-module GitlabExperiment
-  module Generators
-    class InstallGenerator < Rails::Generators::Base
-      source_root File.expand_path(__dir__)
-
-      desc 'Installs the Gitlab Experiment initializer into your application.'
-
-      def copy_initializers
-        copy_file 'templates/initializer.rb', 'config/initializers/gitlab_experiment.rb'
-      end
-
-      def display_post_install
-        readme 'POST_INSTALL' if behavior == :invoke
-      end
-    end
-  end
-end