gitlab-experiment 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module Gitlab
6
+ module Generators
7
+ class ExperimentGenerator < Rails::Generators::NamedBase
8
+ source_root File.expand_path('templates/', __dir__)
9
+ check_class_collision suffix: 'Experiment'
10
+
11
+ argument :variants,
12
+ type: :array,
13
+ default: %w[control candidate],
14
+ banner: 'variant variant'
15
+
16
+ def create_experiment
17
+ template 'experiment.rb', File.join('app/experiments', class_path, "#{file_name}_experiment.rb")
18
+ end
19
+
20
+ hook_for :test_framework
21
+
22
+ private
23
+
24
+ def file_name
25
+ @_file_name ||= remove_possible_suffix(super)
26
+ end
27
+
28
+ def remove_possible_suffix(name)
29
+ name.sub(/_?exp[ei]riment$/i, "") # be somewhat forgiving with spelling
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators'
4
+
5
+ module Gitlab
6
+ module Generators
7
+ module Experiment
8
+ class InstallGenerator < Rails::Generators::Base
9
+ source_root File.expand_path('templates', __dir__)
10
+
11
+ desc 'Installs the Gitlab::Experiment initializer and optional ApplicationExperiment into your application.'
12
+
13
+ class_option :skip_initializer,
14
+ type: :boolean,
15
+ default: false,
16
+ desc: 'Skip the initializer with default configuration'
17
+
18
+ class_option :skip_baseclass,
19
+ type: :boolean,
20
+ default: false,
21
+ desc: 'Skip the ApplicationExperiment base class'
22
+
23
+ def create_initializer
24
+ return if options[:skip_initializer]
25
+
26
+ template 'initializer.rb', 'config/initializers/gitlab_experiment.rb'
27
+ end
28
+
29
+ def create_baseclass
30
+ return if options[:skip_baseclass]
31
+
32
+ template 'application_experiment.rb', 'app/experiments/application_experiment.rb'
33
+ end
34
+
35
+ def display_post_install
36
+ readme 'POST_INSTALL' if behavior == :invoke
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,2 @@
1
+ Gitlab::Experiment has been installed. You may want to adjust the configuration
2
+ that's been provided in the Rails initializer.
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ class ApplicationExperiment < Gitlab::Experiment
4
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ Gitlab::Experiment.configure do |config|
4
+ # Prefix all experiment names with a given value. Use `nil` for none.
5
+ config.name_prefix = nil
6
+
7
+ # The logger is used to log various details of the experiments.
8
+ config.logger = Logger.new($stdout)
9
+
10
+ # The base class that should be instantiated for basic experiments.
11
+ config.base_class = 'ApplicationExperiment'
12
+
13
+ # The caching layer is expected to respond to fetch, like Rails.cache.
14
+ config.cache = nil
15
+
16
+ # Logic this project uses to resolve a variant for a given experiment.
17
+ #
18
+ # This can return an instance of any object that responds to `name`, or can
19
+ # return a variant name as a string, in which case the build in variant
20
+ # class will be used.
21
+ #
22
+ # This block will be executed within the scope of the experiment instance,
23
+ # so can easily access experiment methods, like getting the name or context.
24
+ config.variant_resolver = lambda do |requested_variant|
25
+ # Run the control, unless a variant was requested in code:
26
+ requested_variant || 'control'
27
+
28
+ # Run the candidate, unless a variant was requested, with a fallback:
29
+ #
30
+ # requested_variant || variant_names.first || 'control'
31
+
32
+ # Using Unleash to determine the variant:
33
+ #
34
+ # fallback = Unleash::Variant.new(name: requested_variant || 'control', enabled: true)
35
+ # Unleash.get_variant(name, context.value, fallback)
36
+
37
+ # Using Flipper to determine the variant:
38
+ #
39
+ # TODO: provide example.
40
+ # Variant.new(name: requested_variant || 'control')
41
+ end
42
+
43
+ # Tracking behavior can be implemented to link an event to an experiment.
44
+ #
45
+ # Similar to the variant_resolver, this is called within the scope of the
46
+ # experiment instance and so can access any methods on the experiment,
47
+ # such as name and signature.
48
+ config.tracking_behavior = lambda do |event, args|
49
+ # An example of using a generic logger to track events:
50
+ config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
51
+
52
+ # Using something like snowplow to track events (in gitlab):
53
+ #
54
+ # Gitlab::Tracking.event(name, event, **args.merge(
55
+ # context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
56
+ # 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-2-0', signature
57
+ # )
58
+ # ))
59
+ end
60
+
61
+ # Called at the end of every experiment run, with the result.
62
+ #
63
+ # You may want to track that you've assigned a variant to a given context,
64
+ # or push the experiment into the client or publish results elsewhere, like
65
+ # into redis. Also called within the scope of the experiment instance.
66
+ config.publishing_behavior = lambda do |result|
67
+ # Track the event using our own configured tracking logic.
68
+ track(:assignment)
69
+
70
+ # Push the experiment knowledge into the front end. The signature contains
71
+ # the context key, and the variant that has been determined.
72
+ #
73
+ # Gon.push({ experiment: { name => signature } }, true)
74
+
75
+ # Log using our logging system, so the result (which can be large) can be
76
+ # reviewed later if we want to.
77
+ #
78
+ # Lograge::Event.log(experiment: name, result: result, signature: signature)
79
+ end
80
+
81
+ # Algorithm that consistently generates a hash key for a given hash map.
82
+ #
83
+ # Given a specific context hash map, we need to generate a consistent hash
84
+ # key. The logic in here will be used for generating cache keys, and may also
85
+ # be used when determining which variant may be presented.
86
+ config.context_hash_strategy = lambda do |context|
87
+ values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
88
+ Digest::MD5.hexdigest((context.keys + values).join('|'))
89
+ end
90
+
91
+ # The domain for which this cookie applies so you can restrict to the domain level.
92
+ #
93
+ # When not set, it uses the current host. If you want to provide specific hosts, you can
94
+ # provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
95
+ config.cookie_domain = :all
96
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ <% if namespaced? -%>
4
+ require_dependency "<%= namespaced_path %>/application_experiment"
5
+
6
+ <% end -%>
7
+ <% module_namespacing do -%>
8
+ class <%= class_name %>Experiment < ApplicationExperiment
9
+ <% variants.each do |variant| -%>
10
+ def <%= variant %>_behavior
11
+ end
12
+ <%= "\n" unless variant == variants.last -%>
13
+ <% end -%>
14
+ end
15
+ <% end -%>
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'generators/rspec'
4
+
5
+ module Rspec
6
+ module Generators
7
+ class ExperimentGenerator < Rspec::Generators::Base
8
+ source_root File.expand_path('templates/', __dir__)
9
+
10
+ def create_experiment_spec
11
+ template 'experiment_spec.rb', File.join('spec/experiments', class_path, "#{file_name}_experiment_spec.rb")
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails_helper'
4
+
5
+ <% module_namespacing do -%>
6
+ RSpec.describe <%= class_name %>Experiment do
7
+ pending "add some examples to (or delete) #{__FILE__}"
8
+ end
9
+ <% end -%>
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rails/generators/test_unit'
4
+
5
+ module TestUnit # :nodoc:
6
+ module Generators # :nodoc:
7
+ class ExperimentGenerator < TestUnit::Generators::Base # :nodoc:
8
+ source_root File.expand_path('templates/', __dir__)
9
+
10
+ check_class_collision suffix: 'Test'
11
+
12
+ def create_test_file
13
+ template 'experiment_test.rb', File.join('test/experiments', class_path, "#{file_name}_experiment_test.rb")
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'test_helper'
4
+
5
+ <% module_namespacing do -%>
6
+ class <%= class_name %>ExperimentTest < ActiveSupport::TestCase
7
+ # test "the truth" do
8
+ # assert true
9
+ # end
10
+ end
11
+ <% end -%>
@@ -1,8 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'scientist'
4
+ require 'active_support/callbacks'
5
+ require 'active_support/core_ext/object/blank'
6
+ require 'active_support/core_ext/string/inflections'
4
7
 
8
+ require 'gitlab/experiment/caching'
9
+ require 'gitlab/experiment/callbacks'
5
10
  require 'gitlab/experiment/configuration'
11
+ require 'gitlab/experiment/cookies'
6
12
  require 'gitlab/experiment/context'
7
13
  require 'gitlab/experiment/dsl'
8
14
  require 'gitlab/experiment/variant'
@@ -12,50 +18,91 @@ require 'gitlab/experiment/engine' if defined?(Rails::Engine)
12
18
  module Gitlab
13
19
  class Experiment
14
20
  include Scientist::Experiment
21
+ include Caching
22
+ include Callbacks
15
23
 
16
24
  class << self
17
25
  def configure
18
26
  yield Configuration
19
27
  end
20
28
 
21
- def run(name, variant_name = nil, **context, &block)
22
- instance = new(name, variant_name, **context, &block)
29
+ def run(name = nil, variant_name = nil, **context, &block)
30
+ raise ArgumentError, 'name is required' if name.nil? && base?
31
+
32
+ instance = constantize(name).new(name, variant_name, **context, &block)
23
33
  return instance unless block_given?
24
34
 
25
35
  instance.context.frozen? ? instance.run : instance.tap(&:run)
26
36
  end
37
+
38
+ def experiment_name(name = nil, suffix: true, suffix_word: 'experiment')
39
+ name = (name.presence || self.name).to_s.underscore.sub(%r{(?<char>[_/]|)#{suffix_word}$}, '')
40
+ name = "#{name}#{Regexp.last_match(:char) || '_'}#{suffix_word}"
41
+ suffix ? name : name.sub(/_#{suffix_word}$/, '')
42
+ end
43
+
44
+ def base?
45
+ self == Gitlab::Experiment || name == Configuration.base_class
46
+ end
47
+
48
+ private
49
+
50
+ def constantize(name = nil)
51
+ return self if name.nil?
52
+
53
+ experiment_name(name).classify.safe_constantize || Configuration.base_class.constantize
54
+ end
27
55
  end
28
56
 
29
- delegate :signature, to: :context
57
+ def initialize(name = nil, variant_name = nil, **context)
58
+ raise ArgumentError, 'name is required' if name.blank? && self.class.base?
30
59
 
31
- def initialize(name, variant_name = nil, **context)
32
- @name = name
60
+ @name = self.class.experiment_name(name, suffix: false)
33
61
  @variant_name = variant_name
34
- @context = Context.new(self)
35
-
36
- context(context)
62
+ @excluded = []
63
+ @context = Context.new(self, context)
37
64
 
38
- ignore { true }
65
+ exclude { !@context.trackable? }
39
66
  compare { false }
40
67
 
41
68
  yield self if block_given?
42
69
  end
43
70
 
44
71
  def context(value = nil)
45
- return @context if value.nil?
72
+ return @context if value.blank?
46
73
 
47
74
  @context.value(value)
48
75
  @context
49
76
  end
50
77
 
51
78
  def variant(value = nil)
52
- @variant_name = value unless value.nil?
79
+ return @variant_name = value unless value.nil?
80
+
53
81
  result = instance_exec(@variant_name, &Configuration.variant_resolver)
54
82
  result.respond_to?(:name) ? result : Variant.new(name: result.to_s)
55
83
  end
56
84
 
57
- def run
58
- @result ||= super(variant.name)
85
+ def exclude(&block)
86
+ @excluded << block
87
+ end
88
+
89
+ def run(variant_name = nil)
90
+ @result ||= begin
91
+ @variant_name = variant_name unless variant_name.nil?
92
+ @variant_name ||= :control if excluded?
93
+
94
+ chain = variant_assigned? ? :unsegmented_run : :segmented_run
95
+ run_callbacks(chain) do
96
+ variant_name = cache { variant.name }
97
+
98
+ method_name = "#{variant_name}_behavior"
99
+ if respond_to?(method_name)
100
+ behaviors[variant_name] ||= -> { send(method_name) } # rubocop:disable GitlabSecurity/PublicSend
101
+ end
102
+
103
+ super(variant_name)
104
+ end
105
+ end
59
106
  end
60
107
 
61
108
  def publish(result)
@@ -63,6 +110,8 @@ module Gitlab
63
110
  end
64
111
 
65
112
  def track(action, **event_args)
113
+ return if excluded?
114
+
66
115
  instance_exec(action, event_args, &Configuration.tracking_behavior)
67
116
  end
68
117
 
@@ -71,13 +120,38 @@ module Gitlab
71
120
  end
72
121
 
73
122
  def variant_names
74
- @variant_names = behaviors.keys.tap { |keys| keys.delete('control') }.map(&:to_sym)
123
+ @variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
124
+ end
125
+
126
+ def signature
127
+ { variant: variant.name, experiment: name }.merge(context.signature)
75
128
  end
76
129
 
77
130
  def enabled?
78
131
  true
79
132
  end
80
133
 
134
+ def excluded?
135
+ @excluded.any? { |exclude| exclude.call(self) }
136
+ end
137
+
138
+ def variant_assigned?
139
+ !@variant_name.nil?
140
+ end
141
+
142
+ def id
143
+ "#{name}:#{signature[:key]}"
144
+ end
145
+ alias_method :session_id, :id
146
+
147
+ def flipper_id
148
+ "Experiment;#{id}"
149
+ end
150
+
151
+ def key_for(hash)
152
+ instance_exec(hash, &Configuration.context_hash_strategy)
153
+ end
154
+
81
155
  protected
82
156
 
83
157
  def generate_result(variant_name)
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gitlab
4
+ class Experiment
5
+ module Caching
6
+ def cache(&block)
7
+ return yield unless (cache = Configuration.cache)
8
+
9
+ key, migrations = cache_strategy
10
+ migrated_cache(cache, migrations || [], key) or cache.fetch(key, &block)
11
+ end
12
+
13
+ private
14
+
15
+ def cache_strategy
16
+ [
17
+ "#{name}:#{signature[:key]}",
18
+ signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
19
+ ]
20
+ end
21
+
22
+ def migrated_cache(cache, migrations, new_key)
23
+ migrations.find do |old_key|
24
+ next unless (value = cache.read(old_key))
25
+
26
+ cache.write(new_key, value)
27
+ cache.delete(old_key)
28
+ break value
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end