gitlab-experiment 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +218 -82
- data/lib/generators/gitlab/experiment/USAGE +17 -0
- data/lib/generators/gitlab/experiment/experiment_generator.rb +33 -0
- data/lib/generators/gitlab/experiment/install/install_generator.rb +41 -0
- data/lib/generators/gitlab/experiment/install/templates/POST_INSTALL +2 -0
- data/lib/generators/gitlab/experiment/install/templates/application_experiment.rb.tt +4 -0
- data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt +96 -0
- data/lib/generators/gitlab/experiment/templates/experiment.rb.tt +15 -0
- data/lib/generators/rspec/experiment/experiment_generator.rb +15 -0
- data/lib/generators/rspec/experiment/templates/experiment_spec.rb.tt +9 -0
- data/lib/generators/test_unit/experiment/experiment_generator.rb +17 -0
- data/lib/generators/test_unit/experiment/templates/experiment_test.rb.tt +11 -0
- data/lib/gitlab/experiment.rb +88 -14
- data/lib/gitlab/experiment/caching.rb +33 -0
- data/lib/gitlab/experiment/callbacks.rb +39 -0
- data/lib/gitlab/experiment/configuration.rb +29 -9
- data/lib/gitlab/experiment/context.rb +29 -46
- data/lib/gitlab/experiment/cookies.rb +48 -0
- data/lib/gitlab/experiment/version.rb +1 -1
- metadata +36 -11
- data/lib/generators/gitlab_experiment/install/POST_INSTALL +0 -0
- data/lib/generators/gitlab_experiment/install/install_generator.rb +0 -21
- data/lib/generators/gitlab_experiment/install/templates/initializer.rb +0 -77
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Generators
|
7
|
+
class ExperimentGenerator < Rails::Generators::NamedBase
|
8
|
+
source_root File.expand_path('templates/', __dir__)
|
9
|
+
check_class_collision suffix: 'Experiment'
|
10
|
+
|
11
|
+
argument :variants,
|
12
|
+
type: :array,
|
13
|
+
default: %w[control candidate],
|
14
|
+
banner: 'variant variant'
|
15
|
+
|
16
|
+
def create_experiment
|
17
|
+
template 'experiment.rb', File.join('app/experiments', class_path, "#{file_name}_experiment.rb")
|
18
|
+
end
|
19
|
+
|
20
|
+
hook_for :test_framework
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def file_name
|
25
|
+
@_file_name ||= remove_possible_suffix(super)
|
26
|
+
end
|
27
|
+
|
28
|
+
def remove_possible_suffix(name)
|
29
|
+
name.sub(/_?exp[ei]riment$/i, "") # be somewhat forgiving with spelling
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators'
|
4
|
+
|
5
|
+
module Gitlab
|
6
|
+
module Generators
|
7
|
+
module Experiment
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
9
|
+
source_root File.expand_path('templates', __dir__)
|
10
|
+
|
11
|
+
desc 'Installs the Gitlab::Experiment initializer and optional ApplicationExperiment into your application.'
|
12
|
+
|
13
|
+
class_option :skip_initializer,
|
14
|
+
type: :boolean,
|
15
|
+
default: false,
|
16
|
+
desc: 'Skip the initializer with default configuration'
|
17
|
+
|
18
|
+
class_option :skip_baseclass,
|
19
|
+
type: :boolean,
|
20
|
+
default: false,
|
21
|
+
desc: 'Skip the ApplicationExperiment base class'
|
22
|
+
|
23
|
+
def create_initializer
|
24
|
+
return if options[:skip_initializer]
|
25
|
+
|
26
|
+
template 'initializer.rb', 'config/initializers/gitlab_experiment.rb'
|
27
|
+
end
|
28
|
+
|
29
|
+
def create_baseclass
|
30
|
+
return if options[:skip_baseclass]
|
31
|
+
|
32
|
+
template 'application_experiment.rb', 'app/experiments/application_experiment.rb'
|
33
|
+
end
|
34
|
+
|
35
|
+
def display_post_install
|
36
|
+
readme 'POST_INSTALL' if behavior == :invoke
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
Gitlab::Experiment.configure do |config|
|
4
|
+
# Prefix all experiment names with a given value. Use `nil` for none.
|
5
|
+
config.name_prefix = nil
|
6
|
+
|
7
|
+
# The logger is used to log various details of the experiments.
|
8
|
+
config.logger = Logger.new($stdout)
|
9
|
+
|
10
|
+
# The base class that should be instantiated for basic experiments.
|
11
|
+
config.base_class = 'ApplicationExperiment'
|
12
|
+
|
13
|
+
# The caching layer is expected to respond to fetch, like Rails.cache.
|
14
|
+
config.cache = nil
|
15
|
+
|
16
|
+
# Logic this project uses to resolve a variant for a given experiment.
|
17
|
+
#
|
18
|
+
# This can return an instance of any object that responds to `name`, or can
|
19
|
+
# return a variant name as a string, in which case the build in variant
|
20
|
+
# class will be used.
|
21
|
+
#
|
22
|
+
# This block will be executed within the scope of the experiment instance,
|
23
|
+
# so can easily access experiment methods, like getting the name or context.
|
24
|
+
config.variant_resolver = lambda do |requested_variant|
|
25
|
+
# Run the control, unless a variant was requested in code:
|
26
|
+
requested_variant || 'control'
|
27
|
+
|
28
|
+
# Run the candidate, unless a variant was requested, with a fallback:
|
29
|
+
#
|
30
|
+
# requested_variant || variant_names.first || 'control'
|
31
|
+
|
32
|
+
# Using Unleash to determine the variant:
|
33
|
+
#
|
34
|
+
# fallback = Unleash::Variant.new(name: requested_variant || 'control', enabled: true)
|
35
|
+
# Unleash.get_variant(name, context.value, fallback)
|
36
|
+
|
37
|
+
# Using Flipper to determine the variant:
|
38
|
+
#
|
39
|
+
# TODO: provide example.
|
40
|
+
# Variant.new(name: requested_variant || 'control')
|
41
|
+
end
|
42
|
+
|
43
|
+
# Tracking behavior can be implemented to link an event to an experiment.
|
44
|
+
#
|
45
|
+
# Similar to the variant_resolver, this is called within the scope of the
|
46
|
+
# experiment instance and so can access any methods on the experiment,
|
47
|
+
# such as name and signature.
|
48
|
+
config.tracking_behavior = lambda do |event, args|
|
49
|
+
# An example of using a generic logger to track events:
|
50
|
+
config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
|
51
|
+
|
52
|
+
# Using something like snowplow to track events (in gitlab):
|
53
|
+
#
|
54
|
+
# Gitlab::Tracking.event(name, event, **args.merge(
|
55
|
+
# context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
|
56
|
+
# 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-2-0', signature
|
57
|
+
# )
|
58
|
+
# ))
|
59
|
+
end
|
60
|
+
|
61
|
+
# Called at the end of every experiment run, with the result.
|
62
|
+
#
|
63
|
+
# You may want to track that you've assigned a variant to a given context,
|
64
|
+
# or push the experiment into the client or publish results elsewhere, like
|
65
|
+
# into redis. Also called within the scope of the experiment instance.
|
66
|
+
config.publishing_behavior = lambda do |result|
|
67
|
+
# Track the event using our own configured tracking logic.
|
68
|
+
track(:assignment)
|
69
|
+
|
70
|
+
# Push the experiment knowledge into the front end. The signature contains
|
71
|
+
# the context key, and the variant that has been determined.
|
72
|
+
#
|
73
|
+
# Gon.push({ experiment: { name => signature } }, true)
|
74
|
+
|
75
|
+
# Log using our logging system, so the result (which can be large) can be
|
76
|
+
# reviewed later if we want to.
|
77
|
+
#
|
78
|
+
# Lograge::Event.log(experiment: name, result: result, signature: signature)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Algorithm that consistently generates a hash key for a given hash map.
|
82
|
+
#
|
83
|
+
# Given a specific context hash map, we need to generate a consistent hash
|
84
|
+
# key. The logic in here will be used for generating cache keys, and may also
|
85
|
+
# be used when determining which variant may be presented.
|
86
|
+
config.context_hash_strategy = lambda do |context|
|
87
|
+
values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
|
88
|
+
Digest::MD5.hexdigest((context.keys + values).join('|'))
|
89
|
+
end
|
90
|
+
|
91
|
+
# The domain for which this cookie applies so you can restrict to the domain level.
|
92
|
+
#
|
93
|
+
# When not set, it uses the current host. If you want to provide specific hosts, you can
|
94
|
+
# provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
|
95
|
+
config.cookie_domain = :all
|
96
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
<% if namespaced? -%>
|
4
|
+
require_dependency "<%= namespaced_path %>/application_experiment"
|
5
|
+
|
6
|
+
<% end -%>
|
7
|
+
<% module_namespacing do -%>
|
8
|
+
class <%= class_name %>Experiment < ApplicationExperiment
|
9
|
+
<% variants.each do |variant| -%>
|
10
|
+
def <%= variant %>_behavior
|
11
|
+
end
|
12
|
+
<%= "\n" unless variant == variants.last -%>
|
13
|
+
<% end -%>
|
14
|
+
end
|
15
|
+
<% end -%>
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'generators/rspec'
|
4
|
+
|
5
|
+
module Rspec
|
6
|
+
module Generators
|
7
|
+
class ExperimentGenerator < Rspec::Generators::Base
|
8
|
+
source_root File.expand_path('templates/', __dir__)
|
9
|
+
|
10
|
+
def create_experiment_spec
|
11
|
+
template 'experiment_spec.rb', File.join('spec/experiments', class_path, "#{file_name}_experiment_spec.rb")
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rails/generators/test_unit'
|
4
|
+
|
5
|
+
module TestUnit # :nodoc:
|
6
|
+
module Generators # :nodoc:
|
7
|
+
class ExperimentGenerator < TestUnit::Generators::Base # :nodoc:
|
8
|
+
source_root File.expand_path('templates/', __dir__)
|
9
|
+
|
10
|
+
check_class_collision suffix: 'Test'
|
11
|
+
|
12
|
+
def create_test_file
|
13
|
+
template 'experiment_test.rb', File.join('test/experiments', class_path, "#{file_name}_experiment_test.rb")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/gitlab/experiment.rb
CHANGED
@@ -1,8 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'scientist'
|
4
|
+
require 'active_support/callbacks'
|
5
|
+
require 'active_support/core_ext/object/blank'
|
6
|
+
require 'active_support/core_ext/string/inflections'
|
4
7
|
|
8
|
+
require 'gitlab/experiment/caching'
|
9
|
+
require 'gitlab/experiment/callbacks'
|
5
10
|
require 'gitlab/experiment/configuration'
|
11
|
+
require 'gitlab/experiment/cookies'
|
6
12
|
require 'gitlab/experiment/context'
|
7
13
|
require 'gitlab/experiment/dsl'
|
8
14
|
require 'gitlab/experiment/variant'
|
@@ -12,50 +18,91 @@ require 'gitlab/experiment/engine' if defined?(Rails::Engine)
|
|
12
18
|
module Gitlab
|
13
19
|
class Experiment
|
14
20
|
include Scientist::Experiment
|
21
|
+
include Caching
|
22
|
+
include Callbacks
|
15
23
|
|
16
24
|
class << self
|
17
25
|
def configure
|
18
26
|
yield Configuration
|
19
27
|
end
|
20
28
|
|
21
|
-
def run(name, variant_name = nil, **context, &block)
|
22
|
-
|
29
|
+
def run(name = nil, variant_name = nil, **context, &block)
|
30
|
+
raise ArgumentError, 'name is required' if name.nil? && base?
|
31
|
+
|
32
|
+
instance = constantize(name).new(name, variant_name, **context, &block)
|
23
33
|
return instance unless block_given?
|
24
34
|
|
25
35
|
instance.context.frozen? ? instance.run : instance.tap(&:run)
|
26
36
|
end
|
37
|
+
|
38
|
+
def experiment_name(name = nil, suffix: true, suffix_word: 'experiment')
|
39
|
+
name = (name.presence || self.name).to_s.underscore.sub(%r{(?<char>[_/]|)#{suffix_word}$}, '')
|
40
|
+
name = "#{name}#{Regexp.last_match(:char) || '_'}#{suffix_word}"
|
41
|
+
suffix ? name : name.sub(/_#{suffix_word}$/, '')
|
42
|
+
end
|
43
|
+
|
44
|
+
def base?
|
45
|
+
self == Gitlab::Experiment || name == Configuration.base_class
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def constantize(name = nil)
|
51
|
+
return self if name.nil?
|
52
|
+
|
53
|
+
experiment_name(name).classify.safe_constantize || Configuration.base_class.constantize
|
54
|
+
end
|
27
55
|
end
|
28
56
|
|
29
|
-
|
57
|
+
def initialize(name = nil, variant_name = nil, **context)
|
58
|
+
raise ArgumentError, 'name is required' if name.blank? && self.class.base?
|
30
59
|
|
31
|
-
|
32
|
-
@name = name
|
60
|
+
@name = self.class.experiment_name(name, suffix: false)
|
33
61
|
@variant_name = variant_name
|
34
|
-
@
|
35
|
-
|
36
|
-
context(context)
|
62
|
+
@excluded = []
|
63
|
+
@context = Context.new(self, context)
|
37
64
|
|
38
|
-
|
65
|
+
exclude { !@context.trackable? }
|
39
66
|
compare { false }
|
40
67
|
|
41
68
|
yield self if block_given?
|
42
69
|
end
|
43
70
|
|
44
71
|
def context(value = nil)
|
45
|
-
return @context if value.
|
72
|
+
return @context if value.blank?
|
46
73
|
|
47
74
|
@context.value(value)
|
48
75
|
@context
|
49
76
|
end
|
50
77
|
|
51
78
|
def variant(value = nil)
|
52
|
-
@variant_name = value unless value.nil?
|
79
|
+
return @variant_name = value unless value.nil?
|
80
|
+
|
53
81
|
result = instance_exec(@variant_name, &Configuration.variant_resolver)
|
54
82
|
result.respond_to?(:name) ? result : Variant.new(name: result.to_s)
|
55
83
|
end
|
56
84
|
|
57
|
-
def
|
58
|
-
@
|
85
|
+
def exclude(&block)
|
86
|
+
@excluded << block
|
87
|
+
end
|
88
|
+
|
89
|
+
def run(variant_name = nil)
|
90
|
+
@result ||= begin
|
91
|
+
@variant_name = variant_name unless variant_name.nil?
|
92
|
+
@variant_name ||= :control if excluded?
|
93
|
+
|
94
|
+
chain = variant_assigned? ? :unsegmented_run : :segmented_run
|
95
|
+
run_callbacks(chain) do
|
96
|
+
variant_name = cache { variant.name }
|
97
|
+
|
98
|
+
method_name = "#{variant_name}_behavior"
|
99
|
+
if respond_to?(method_name)
|
100
|
+
behaviors[variant_name] ||= -> { send(method_name) } # rubocop:disable GitlabSecurity/PublicSend
|
101
|
+
end
|
102
|
+
|
103
|
+
super(variant_name)
|
104
|
+
end
|
105
|
+
end
|
59
106
|
end
|
60
107
|
|
61
108
|
def publish(result)
|
@@ -63,6 +110,8 @@ module Gitlab
|
|
63
110
|
end
|
64
111
|
|
65
112
|
def track(action, **event_args)
|
113
|
+
return if excluded?
|
114
|
+
|
66
115
|
instance_exec(action, event_args, &Configuration.tracking_behavior)
|
67
116
|
end
|
68
117
|
|
@@ -71,13 +120,38 @@ module Gitlab
|
|
71
120
|
end
|
72
121
|
|
73
122
|
def variant_names
|
74
|
-
@variant_names
|
123
|
+
@variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
|
124
|
+
end
|
125
|
+
|
126
|
+
def signature
|
127
|
+
{ variant: variant.name, experiment: name }.merge(context.signature)
|
75
128
|
end
|
76
129
|
|
77
130
|
def enabled?
|
78
131
|
true
|
79
132
|
end
|
80
133
|
|
134
|
+
def excluded?
|
135
|
+
@excluded.any? { |exclude| exclude.call(self) }
|
136
|
+
end
|
137
|
+
|
138
|
+
def variant_assigned?
|
139
|
+
!@variant_name.nil?
|
140
|
+
end
|
141
|
+
|
142
|
+
def id
|
143
|
+
"#{name}:#{signature[:key]}"
|
144
|
+
end
|
145
|
+
alias_method :session_id, :id
|
146
|
+
|
147
|
+
def flipper_id
|
148
|
+
"Experiment;#{id}"
|
149
|
+
end
|
150
|
+
|
151
|
+
def key_for(hash)
|
152
|
+
instance_exec(hash, &Configuration.context_hash_strategy)
|
153
|
+
end
|
154
|
+
|
81
155
|
protected
|
82
156
|
|
83
157
|
def generate_result(variant_name)
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
class Experiment
|
5
|
+
module Caching
|
6
|
+
def cache(&block)
|
7
|
+
return yield unless (cache = Configuration.cache)
|
8
|
+
|
9
|
+
key, migrations = cache_strategy
|
10
|
+
migrated_cache(cache, migrations || [], key) or cache.fetch(key, &block)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def cache_strategy
|
16
|
+
[
|
17
|
+
"#{name}:#{signature[:key]}",
|
18
|
+
signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
|
19
|
+
]
|
20
|
+
end
|
21
|
+
|
22
|
+
def migrated_cache(cache, migrations, new_key)
|
23
|
+
migrations.find do |old_key|
|
24
|
+
next unless (value = cache.read(old_key))
|
25
|
+
|
26
|
+
cache.write(new_key, value)
|
27
|
+
cache.delete(old_key)
|
28
|
+
break value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|