gitlab-experiment 0.4.1 → 0.4.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +140 -50
- data/lib/generators/gitlab/experiment/install/templates/initializer.rb.tt +27 -30
- data/lib/gitlab/experiment.rb +60 -35
- data/lib/gitlab/experiment/caching.rb +13 -7
- data/lib/gitlab/experiment/callbacks.rb +18 -15
- data/lib/gitlab/experiment/configuration.rb +9 -11
- data/lib/gitlab/experiment/context.rb +8 -0
- data/lib/gitlab/experiment/cookies.rb +1 -1
- data/lib/gitlab/experiment/rspec_matchers.rb +91 -0
- data/lib/gitlab/experiment/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 51c0f257bd0cea5fa7b970e764ac5cce9f24c5a844673f05579d1c902a46fc39
|
4
|
+
data.tar.gz: ddd93dc46cdde532f20edd5209651d511105157ee9dd13d58c5a60cb229cd3ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 243606b6b55ebe069296c564ad801a6b25075a85d2b2ca38eb47e836b75b655fa9472fbcf0a0cb01c83093b97d529f8b2aec04389d79119064b5021c975b0042
|
7
|
+
data.tar.gz: 1900d8b44458d2cf1e2086cc48d5e3ccc19160926c1dc9b827b7a93d042edf7354e1eeb35cae12e0c33f133aef39f605dbd5446edf987c1d99a38264d3b97d72
|
data/README.md
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
GitLab Experiment
|
2
|
+
=================
|
2
3
|
|
3
4
|
<img alt="experiment" src="/uploads/60990b2dbf4c0406bbf8b7f998de2dea/experiment.png" align="right" width="40%">
|
4
5
|
|
@@ -16,6 +17,8 @@ When we discuss the behavior of this gem, we'll use terms like experiment, conte
|
|
16
17
|
|
17
18
|
Candidate and variant are the same concept, but simplify how we speak about experimental paths.<br clear="all">
|
18
19
|
|
20
|
+
[[_TOC_]]
|
21
|
+
|
19
22
|
## Installation
|
20
23
|
|
21
24
|
Add the gem to your Gemfile and then `bundle install`.
|
@@ -24,7 +27,7 @@ Add the gem to your Gemfile and then `bundle install`.
|
|
24
27
|
gem 'gitlab-experiment'
|
25
28
|
```
|
26
29
|
|
27
|
-
If you're using Rails, you can install the initializer
|
30
|
+
If you're using Rails, you can install the initializer which provides basic configuration, documentation, and the base experiment class that all your experiments can inherit from.
|
28
31
|
|
29
32
|
```shell
|
30
33
|
$ rails generate gitlab:experiment:install
|
@@ -79,72 +82,91 @@ To this end, we track events that are important by calling the same experiment e
|
|
79
82
|
experiment(:notification_toggle, actor: user).track(:clicked_button)
|
80
83
|
```
|
81
84
|
|
82
|
-
|
83
|
-
<summary>You can also use the more low level class or instance interfaces...</summary>
|
85
|
+
### Custom experiments
|
84
86
|
|
85
|
-
|
87
|
+
You can craft more advanced behaviors by defining custom experiments at a higher level. To do this you can define a class that inherits from `ApplicationExperiment` (or `Gitlab::Experiment`).
|
86
88
|
|
87
|
-
|
88
|
-
exp = Gitlab::Experiment.run(:notification_toggle, actor: user) do |e|
|
89
|
-
# Context may be passed in the block, but must be finalized before calling
|
90
|
-
# run or track.
|
91
|
-
e.context(project: project) # add the project to the context
|
89
|
+
Let's say you want to do more advanced segmentation, or provide default behavior for the variants on the experiment we've already outlined above -- that way if the variants aren't defined in the block at the time the experiment is run, these methods will be used.
|
92
90
|
|
93
|
-
|
94
|
-
e.use { render_toggle } # control
|
95
|
-
e.try { render_button } # candidate
|
96
|
-
end
|
91
|
+
You can generate a custom experiment by running:
|
97
92
|
|
98
|
-
|
99
|
-
|
100
|
-
```
|
93
|
+
```shell
|
94
|
+
$ rails generate gitlab:experiment NotificationToggle control candidate
|
95
|
+
```
|
101
96
|
|
102
|
-
|
97
|
+
This will generate a file in `app/experiments/notification_toggle_experiment.rb`, as well as a test file for you to further expand on.
|
98
|
+
|
99
|
+
Here are some examples of what you can introduce once you have a custom experiment defined.
|
103
100
|
|
104
101
|
```ruby
|
105
|
-
|
106
|
-
#
|
107
|
-
|
108
|
-
|
102
|
+
class NotificationToggleExperiment < ApplicationExperiment
|
103
|
+
# Exclude any users that aren't me.
|
104
|
+
exclude :all_but_me
|
105
|
+
|
106
|
+
# Segment any account less than 2 weeks old into the candidate, without
|
107
|
+
# asking the variant resolver to decide which variant to provide.
|
108
|
+
segment :account_age, variant: :candidate
|
109
|
+
|
110
|
+
# Define the default control behavior, which can be overridden at
|
111
|
+
# experiment time.
|
112
|
+
def control_behavior
|
113
|
+
# render_toggle
|
114
|
+
end
|
109
115
|
|
110
|
-
# Define the
|
111
|
-
|
112
|
-
|
116
|
+
# Define the default candidate behavior, which can be overridden
|
117
|
+
# at experiment time.
|
118
|
+
def candidate_behavior
|
119
|
+
# render_button
|
120
|
+
end
|
113
121
|
|
114
|
-
|
122
|
+
private
|
123
|
+
|
124
|
+
def all_but_me
|
125
|
+
context.actor&.username == 'jejacks0n'
|
126
|
+
end
|
127
|
+
|
128
|
+
def account_age
|
129
|
+
context.actor && context.actor.created_at < 2.weeks.ago
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# The class will be looked up based on the experiment name provided.
|
134
|
+
exp = experiment(:notification_toggle, actor: user)
|
135
|
+
exp # => instance of NotificationToggleExperiment
|
136
|
+
|
137
|
+
# Run the experiment -- returning the result.
|
115
138
|
exp.run
|
116
139
|
|
117
140
|
# Track an event on the experiment we've defined.
|
118
141
|
exp.track(:clicked_button)
|
119
142
|
```
|
120
143
|
|
121
|
-
|
144
|
+
You can now also do things very similar to the simple examples and override the default variant behaviors defined in the custom experiment -- keeping in mind that this should be carefully considered within the scope of your experiment.
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
experiment(:notification_toggle, actor: user) do |e|
|
148
|
+
e.use { render_special_toggle } # override default control behavior
|
149
|
+
end
|
150
|
+
```
|
122
151
|
|
123
152
|
<details>
|
124
|
-
<summary>You can
|
153
|
+
<summary>You can also use the lower level class interface...</summary>
|
125
154
|
|
126
|
-
###
|
155
|
+
### Using the `.run` approach
|
127
156
|
|
128
|
-
|
129
|
-
class NotificationExperiment < Gitlab::Experiment
|
130
|
-
def initialize(variant_name = nil, **context, &block)
|
131
|
-
super(:notification_toggle, variant_name, **context, &block)
|
157
|
+
This is useful if you haven't included the DSL and so don't have access to the `experiment` method, but still want to execute an experiment. This is ultimately what the `experiment` method calls through to, and the method signatures are the same.
|
132
158
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
159
|
+
```ruby
|
160
|
+
exp = Gitlab::Experiment.run(:notification_toggle, actor: user) do |e|
|
161
|
+
# Context may be passed in the block, but must be finalized before calling
|
162
|
+
# run or track.
|
163
|
+
e.context(project: project) # add the project to the context
|
138
164
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
e.context(project: project) # add the project id to the context
|
165
|
+
# Define the control and candidate variant.
|
166
|
+
e.use { render_toggle } # control
|
167
|
+
e.try { render_button } # candidate
|
143
168
|
end
|
144
169
|
|
145
|
-
# Run the experiment -- returning the result.
|
146
|
-
exp.run
|
147
|
-
|
148
170
|
# Track an event on the experiment we've defined.
|
149
171
|
exp.track(:clicked_button)
|
150
172
|
```
|
@@ -152,17 +174,17 @@ exp.track(:clicked_button)
|
|
152
174
|
</details>
|
153
175
|
|
154
176
|
<details>
|
155
|
-
<summary>You can also specify the variant to use...</summary>
|
177
|
+
<summary>You can also specify the variant to use for segmentation...</summary>
|
156
178
|
|
157
179
|
### Specifying variant
|
158
180
|
|
159
|
-
|
181
|
+
Generally, defining segmentation rules is a better way to approach routing into specific variants, but it's possible to explicitly specify the variant when running an experiment. It's important to know what this might do to your data during rollout, so use this with careful consideration.
|
160
182
|
|
161
183
|
```ruby
|
162
184
|
experiment(:notification_toggle, :no_interface, actor: user) do |e|
|
163
185
|
e.use { render_toggle } # control
|
164
186
|
e.try { render_button } # candidate
|
165
|
-
e.try(:no_interface) { no_interface! } # variant
|
187
|
+
e.try(:no_interface) { no_interface! } # no_interface variant
|
166
188
|
end
|
167
189
|
```
|
168
190
|
|
@@ -188,6 +210,46 @@ end
|
|
188
210
|
|
189
211
|
</details>
|
190
212
|
|
213
|
+
### Segmentation rules
|
214
|
+
|
215
|
+
This library comes with the capability to segment contexts into a specific variant, before asking the variant resolver which variant to provide.
|
216
|
+
|
217
|
+
Segmentation can be achieved by using a custom experiment class and specifying the segmentation rules at a class level.
|
218
|
+
|
219
|
+
```ruby
|
220
|
+
class NotificationToggleExperiment < ApplicationExperiment
|
221
|
+
segment(variant: :variant_one) { context.actor.username == 'jejacks0n' }
|
222
|
+
segment(variant: :variant_two) { context.actor.created_at < 2.weeks.ago }
|
223
|
+
end
|
224
|
+
```
|
225
|
+
|
226
|
+
In the previous examples, any user with the username `'jejacks0n'` would always receive the experience defined in "variant_one". As well, any account less than 2 weeks old would get the alternate experience defined in "variant_two".
|
227
|
+
|
228
|
+
When an experiment is run, the segmentation rules are executed in the order they're defined. The first segmentation rule to produce a truthy result is the one which gets used to assign the variant. The remaining segmentation rules are skipped.
|
229
|
+
|
230
|
+
This means that any user with the name `'jejacks0n'`, regardless of account age, will always be provided the experience as defined in "variant_one".
|
231
|
+
|
232
|
+
### Exclusion rules
|
233
|
+
|
234
|
+
Exclusion rules are similar to segmentation rules, but are intended to determine if a context should even be considered as something we should track events towards. Exclusion means we don't care about the events in relation to the given context.
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
class NotificationToggleExperiment < ApplicationExperiment
|
238
|
+
exclude { context.actor.username != 'jejacks0n' }
|
239
|
+
exclude { context.actor.created_at < 2.weeks.ago }
|
240
|
+
end
|
241
|
+
```
|
242
|
+
|
243
|
+
The previous examples will force all users with a username not matching `'jejacks0n'` and all newish users into the control. No events would be tracked for those.
|
244
|
+
|
245
|
+
You may need to check exclusion in custom tracking logic, by checking `excluded?` or wrapping your code in a callback block:
|
246
|
+
|
247
|
+
```ruby
|
248
|
+
run_callbacks(:exclusion_check) do
|
249
|
+
track(:my_event, value: expensive_method_call)
|
250
|
+
end
|
251
|
+
```
|
252
|
+
|
191
253
|
### Return value
|
192
254
|
|
193
255
|
By default the return value is a `Gitlab::Experiment` instance. In simple cases you may want only the results of the experiment though. You can call `run` within the block to get the return value of the assigned variant.
|
@@ -307,7 +369,7 @@ Gitlab::Experiment.configure do |config|
|
|
307
369
|
end
|
308
370
|
```
|
309
371
|
|
310
|
-
More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb).
|
372
|
+
More examples for configuration are available in the provided [rails initializer](lib/generators/gitlab/experiment/install/templates/initializer.rb.tt).
|
311
373
|
|
312
374
|
### Client layer / JavaScript
|
313
375
|
|
@@ -351,4 +413,32 @@ If you only include a user, that user would get the same experience across every
|
|
351
413
|
|
352
414
|
Each of these approaches could be desirable given the objectives of your experiment.
|
353
415
|
|
354
|
-
|
416
|
+
## Development
|
417
|
+
|
418
|
+
After checking out the repo, run `bundle install` to install dependencies.
|
419
|
+
Then, run `bundle exec rake` to run the tests. You can also run `bundle exec pry` for an
|
420
|
+
interactive prompt that will allow you to experiment.
|
421
|
+
|
422
|
+
## Contributing
|
423
|
+
|
424
|
+
Bug reports and merge requests are welcome on GitLab at
|
425
|
+
https://gitlab.com/gitlab-org/gitlab-experiment. This project is intended to be a
|
426
|
+
safe, welcoming space for collaboration, and contributors are expected to adhere
|
427
|
+
to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
428
|
+
|
429
|
+
## Release Process
|
430
|
+
|
431
|
+
Please refer to the [Release Process](docs/release_process.md).
|
432
|
+
|
433
|
+
## License
|
434
|
+
|
435
|
+
The gem is available as open source under the terms of the
|
436
|
+
[MIT License](http://opensource.org/licenses/MIT).
|
437
|
+
|
438
|
+
## Code of Conduct
|
439
|
+
|
440
|
+
Everyone interacting in the `Gitlab::Experiment` project’s codebases, issue trackers,
|
441
|
+
chat rooms and mailing lists is expected to follow the
|
442
|
+
[code of conduct](CODE_OF_CONDUCT.md).
|
443
|
+
|
444
|
+
***Make code not war***
|
@@ -7,44 +7,41 @@ Gitlab::Experiment.configure do |config|
|
|
7
7
|
# The logger is used to log various details of the experiments.
|
8
8
|
config.logger = Logger.new($stdout)
|
9
9
|
|
10
|
-
# The base class that should be instantiated for basic experiments.
|
10
|
+
# The base class that should be instantiated for basic experiments. It should
|
11
|
+
# be a string, so we can constantize it later.
|
11
12
|
config.base_class = 'ApplicationExperiment'
|
12
13
|
|
13
|
-
# The caching layer is expected to respond to fetch, like Rails.cache
|
14
|
+
# The caching layer is expected to respond to fetch, like Rails.cache for
|
15
|
+
# instance -- or anything that adheres to ActiveSupport::Cache::Store.
|
14
16
|
config.cache = nil
|
15
17
|
|
18
|
+
# The domain to use on cookies.
|
19
|
+
#
|
20
|
+
# When not set, it uses the current host. If you want to provide specific
|
21
|
+
# hosts, you use `:all`, or provide an array like
|
22
|
+
# `['www.gitlab.com', '.gitlab.com']`.
|
23
|
+
config.cookie_domain = :all
|
24
|
+
|
16
25
|
# Logic this project uses to resolve a variant for a given experiment.
|
17
26
|
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
# class will be used.
|
27
|
+
# Should return a symbol or string that represents the variant that should
|
28
|
+
# be assigned. Blank or nil values will be defaulted to the control.
|
21
29
|
#
|
22
|
-
# This block
|
23
|
-
#
|
30
|
+
# This block is executed within the scope of the experiment and so can access
|
31
|
+
# experiment methods, like `name`, `context`, and `signature`.
|
24
32
|
config.variant_resolver = lambda do |requested_variant|
|
25
33
|
# Run the control, unless a variant was requested in code:
|
26
|
-
requested_variant
|
34
|
+
requested_variant
|
27
35
|
|
28
36
|
# Run the candidate, unless a variant was requested, with a fallback:
|
29
37
|
#
|
30
|
-
# requested_variant || variant_names.first ||
|
31
|
-
|
32
|
-
# Using Unleash to determine the variant:
|
33
|
-
#
|
34
|
-
# fallback = Unleash::Variant.new(name: requested_variant || 'control', enabled: true)
|
35
|
-
# Unleash.get_variant(name, context.value, fallback)
|
36
|
-
|
37
|
-
# Using Flipper to determine the variant:
|
38
|
-
#
|
39
|
-
# TODO: provide example.
|
40
|
-
# Variant.new(name: requested_variant || 'control')
|
38
|
+
# requested_variant || variant_names.first || nil
|
41
39
|
end
|
42
40
|
|
43
41
|
# Tracking behavior can be implemented to link an event to an experiment.
|
44
42
|
#
|
45
|
-
#
|
46
|
-
# experiment
|
47
|
-
# such as name and signature.
|
43
|
+
# This block is executed within the scope of the experiment and so can access
|
44
|
+
# experiment methods, like `name`, `context`, and `signature`.
|
48
45
|
config.tracking_behavior = lambda do |event, args|
|
49
46
|
# An example of using a generic logger to track events:
|
50
47
|
config.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
|
@@ -61,8 +58,11 @@ Gitlab::Experiment.configure do |config|
|
|
61
58
|
# Called at the end of every experiment run, with the result.
|
62
59
|
#
|
63
60
|
# You may want to track that you've assigned a variant to a given context,
|
64
|
-
# or push the experiment into the client or publish results elsewhere
|
65
|
-
# into redis.
|
61
|
+
# or push the experiment into the client or publish results elsewhere like
|
62
|
+
# into redis.
|
63
|
+
#
|
64
|
+
# This block is executed within the scope of the experiment and so can access
|
65
|
+
# experiment methods, like `name`, `context`, and `signature`.
|
66
66
|
config.publishing_behavior = lambda do |result|
|
67
67
|
# Track the event using our own configured tracking logic.
|
68
68
|
track(:assignment)
|
@@ -83,14 +83,11 @@ Gitlab::Experiment.configure do |config|
|
|
83
83
|
# Given a specific context hash map, we need to generate a consistent hash
|
84
84
|
# key. The logic in here will be used for generating cache keys, and may also
|
85
85
|
# be used when determining which variant may be presented.
|
86
|
+
#
|
87
|
+
# This block is executed within the scope of the experiment and so can access
|
88
|
+
# experiment methods, like `name`, `context`, and `signature`.
|
86
89
|
config.context_hash_strategy = lambda do |context|
|
87
90
|
values = context.values.map { |v| (v.respond_to?(:to_global_id) ? v.to_global_id : v).to_s }
|
88
91
|
Digest::MD5.hexdigest((context.keys + values).join('|'))
|
89
92
|
end
|
90
|
-
|
91
|
-
# The domain for which this cookie applies so you can restrict to the domain level.
|
92
|
-
#
|
93
|
-
# When not set, it uses the current host. If you want to provide specific hosts, you can
|
94
|
-
# provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
|
95
|
-
config.cookie_domain = :all
|
96
93
|
end
|
data/lib/gitlab/experiment.rb
CHANGED
@@ -30,7 +30,7 @@ module Gitlab
|
|
30
30
|
raise ArgumentError, 'name is required' if name.nil? && base?
|
31
31
|
|
32
32
|
instance = constantize(name).new(name, variant_name, **context, &block)
|
33
|
-
return instance unless
|
33
|
+
return instance unless block
|
34
34
|
|
35
35
|
instance.context.frozen? ? instance.run : instance.tap(&:run)
|
36
36
|
end
|
@@ -58,11 +58,9 @@ module Gitlab
|
|
58
58
|
raise ArgumentError, 'name is required' if name.blank? && self.class.base?
|
59
59
|
|
60
60
|
@name = self.class.experiment_name(name, suffix: false)
|
61
|
-
@
|
62
|
-
@
|
63
|
-
@context = Context.new(self, context)
|
61
|
+
@context = Context.new(self, **context)
|
62
|
+
@variant_name = cache_variant(variant_name) { nil } if variant_name.present?
|
64
63
|
|
65
|
-
exclude { !@context.trackable? }
|
66
64
|
compare { false }
|
67
65
|
|
68
66
|
yield self if block_given?
|
@@ -76,31 +74,31 @@ module Gitlab
|
|
76
74
|
end
|
77
75
|
|
78
76
|
def variant(value = nil)
|
79
|
-
|
77
|
+
if value.blank? && @variant_name || @resolving_variant
|
78
|
+
return Variant.new(name: (@variant_name || :unresolved).to_s)
|
79
|
+
end
|
80
80
|
|
81
|
-
|
82
|
-
result.respond_to?(:name) ? result : Variant.new(name: (result.presence || :control).to_s)
|
83
|
-
end
|
81
|
+
@variant_name = value unless value.blank?
|
84
82
|
|
85
|
-
|
86
|
-
@excluded << block
|
87
|
-
end
|
88
|
-
|
89
|
-
def run(variant_name = nil)
|
90
|
-
@result ||= begin
|
91
|
-
@variant_name = variant_name unless variant_name.nil?
|
83
|
+
if enabled?
|
92
84
|
@variant_name ||= :control if excluded?
|
93
85
|
|
94
|
-
|
95
|
-
|
96
|
-
variant_name =
|
86
|
+
@resolving_variant = true
|
87
|
+
if (result = cache_variant(@variant_name) { resolve_variant_name }).present?
|
88
|
+
@variant_name = result.to_sym
|
89
|
+
end
|
90
|
+
end
|
97
91
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
92
|
+
Variant.new(name: (@variant_name || :control).to_s)
|
93
|
+
ensure
|
94
|
+
@resolving_variant = false
|
95
|
+
end
|
102
96
|
|
103
|
-
|
97
|
+
def run(variant_name = nil)
|
98
|
+
@result ||= begin
|
99
|
+
variant_name = variant(variant_name).name
|
100
|
+
run_callbacks(run_with_segmenting? ? :segmented_run : :unsegmented_run) do
|
101
|
+
super(@variant_name ||= variant_name)
|
104
102
|
end
|
105
103
|
end
|
106
104
|
end
|
@@ -110,7 +108,7 @@ module Gitlab
|
|
110
108
|
end
|
111
109
|
|
112
110
|
def track(action, **event_args)
|
113
|
-
return
|
111
|
+
return unless should_track?
|
114
112
|
|
115
113
|
instance_exec(action, event_args, &Configuration.tracking_behavior)
|
116
114
|
end
|
@@ -123,24 +121,26 @@ module Gitlab
|
|
123
121
|
@variant_names ||= behaviors.keys.map(&:to_sym) - [:control]
|
124
122
|
end
|
125
123
|
|
126
|
-
def
|
127
|
-
|
128
|
-
|
124
|
+
def behaviors
|
125
|
+
@behaviors ||= public_methods.each_with_object(super) do |name, behaviors|
|
126
|
+
next unless name.end_with?('_behavior')
|
129
127
|
|
130
|
-
|
131
|
-
|
128
|
+
behavior_name = name.to_s.sub(/_behavior$/, '')
|
129
|
+
behaviors[behavior_name] ||= -> { send(name) } # rubocop:disable GitlabSecurity/PublicSend
|
130
|
+
end
|
132
131
|
end
|
133
132
|
|
134
|
-
def
|
135
|
-
|
133
|
+
def try(name = nil, &block)
|
134
|
+
name = (name || 'candidate').to_s
|
135
|
+
behaviors[name] = block
|
136
136
|
end
|
137
137
|
|
138
|
-
def
|
139
|
-
|
138
|
+
def signature
|
139
|
+
{ variant: variant.name, experiment: name }.merge(context.signature)
|
140
140
|
end
|
141
141
|
|
142
142
|
def id
|
143
|
-
"#{name}:#{
|
143
|
+
"#{name}:#{key_for(context.value)}"
|
144
144
|
end
|
145
145
|
alias_method :session_id, :id
|
146
146
|
|
@@ -154,6 +154,31 @@ module Gitlab
|
|
154
154
|
|
155
155
|
protected
|
156
156
|
|
157
|
+
def enabled?
|
158
|
+
true
|
159
|
+
end
|
160
|
+
|
161
|
+
def excluded?
|
162
|
+
@excluded ||= !@context.trackable? || # adhere to DNT headers
|
163
|
+
!run_callbacks(:exclusion_check) { :not_excluded } # didn't pass exclusion check
|
164
|
+
end
|
165
|
+
|
166
|
+
def should_track?
|
167
|
+
enabled? && !excluded?
|
168
|
+
end
|
169
|
+
|
170
|
+
def run_with_segmenting?
|
171
|
+
!variant_assigned? && enabled? && !excluded?
|
172
|
+
end
|
173
|
+
|
174
|
+
def variant_assigned?
|
175
|
+
!@variant_name.nil?
|
176
|
+
end
|
177
|
+
|
178
|
+
def resolve_variant_name
|
179
|
+
instance_exec(@variant_name, &Configuration.variant_resolver)
|
180
|
+
end
|
181
|
+
|
157
182
|
def generate_result(variant_name)
|
158
183
|
observation = Scientist::Observation.new(variant_name, self, &behaviors[variant_name])
|
159
184
|
Scientist::Result.new(self, [observation], observation)
|
@@ -3,20 +3,26 @@
|
|
3
3
|
module Gitlab
|
4
4
|
class Experiment
|
5
5
|
module Caching
|
6
|
-
def
|
7
|
-
|
6
|
+
def cache_variant(specified = nil, &block)
|
7
|
+
cache = Configuration.cache
|
8
|
+
return (specified.presence || yield) unless cache
|
8
9
|
|
9
10
|
key, migrations = cache_strategy
|
10
|
-
migrated_cache(cache, migrations || [], key)
|
11
|
+
result = migrated_cache(cache, migrations || [], key) || cache.fetch(key, &block)
|
12
|
+
return result unless specified.present?
|
13
|
+
|
14
|
+
cache.write(cache_key, specified) if result != specified
|
15
|
+
specified
|
16
|
+
end
|
17
|
+
|
18
|
+
def cache_key(key = nil)
|
19
|
+
"#{name}:#{key || context.signature[:key]}"
|
11
20
|
end
|
12
21
|
|
13
22
|
private
|
14
23
|
|
15
24
|
def cache_strategy
|
16
|
-
[
|
17
|
-
"#{name}:#{signature[:key]}",
|
18
|
-
signature[:migration_keys]&.map { |key| "#{name}:#{key}" }
|
19
|
-
]
|
25
|
+
[cache_key, context.signature[:migration_keys]&.map { |key| cache_key(key) }]
|
20
26
|
end
|
21
27
|
|
22
28
|
def migrated_cache(cache, migrations, new_key)
|
@@ -7,31 +7,34 @@ module Gitlab
|
|
7
7
|
include ActiveSupport::Callbacks
|
8
8
|
|
9
9
|
included do
|
10
|
-
define_callbacks(
|
11
|
-
|
12
|
-
|
13
|
-
)
|
14
|
-
|
15
|
-
define_callbacks(
|
16
|
-
:segmented_run,
|
17
|
-
skip_after_callbacks_if_terminated: false,
|
18
|
-
terminator: lambda do |target, result_lambda|
|
19
|
-
result_lambda.call
|
20
|
-
target.variant_assigned?
|
21
|
-
end
|
22
|
-
)
|
10
|
+
define_callbacks(:unsegmented_run)
|
11
|
+
define_callbacks(:segmented_run)
|
12
|
+
define_callbacks(:exclusion_check, skip_after_callbacks_if_terminated: true)
|
23
13
|
end
|
24
14
|
|
25
15
|
class_methods do
|
16
|
+
def exclude(*filter_list, **options, &block)
|
17
|
+
filters = filter_list.unshift(block).compact.map do |filter|
|
18
|
+
result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
|
19
|
+
lambda do |target|
|
20
|
+
throw(:abort) if target.instance_variable_get(:'@excluded') || result_lambda.call(target, nil) == true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
raise ArgumentError, 'no filters provided' if filters.empty?
|
25
|
+
|
26
|
+
set_callback(:exclusion_check, :before, *filters, options)
|
27
|
+
end
|
28
|
+
|
26
29
|
def segment(*filter_list, variant:, **options, &block)
|
27
30
|
filters = filter_list.unshift(block).compact.map do |filter|
|
28
31
|
result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
|
29
|
-
->(target) { target.variant(variant) if result_lambda.call(target, nil) }
|
32
|
+
->(target) { target.variant(variant) if !target.variant_assigned? && result_lambda.call(target, nil) }
|
30
33
|
end
|
31
34
|
|
32
35
|
raise ArgumentError, 'no filters provided' if filters.empty?
|
33
36
|
|
34
|
-
set_callback(:segmented_run, :before, *filters, options
|
37
|
+
set_callback(:segmented_run, :before, *filters, options)
|
35
38
|
end
|
36
39
|
end
|
37
40
|
end
|
@@ -18,12 +18,16 @@ module Gitlab
|
|
18
18
|
# The base class that should be instantiated for basic experiments.
|
19
19
|
@base_class = 'Gitlab::Experiment'
|
20
20
|
|
21
|
-
#
|
21
|
+
# The caching layer is expected to respond to fetch, like Rails.cache.
|
22
22
|
@cache = nil
|
23
23
|
|
24
|
+
# The domain to use on cookies.
|
25
|
+
@cookie_domain = :all
|
26
|
+
|
24
27
|
# Logic this project uses to resolve a variant for a given experiment.
|
28
|
+
# If no variant is determined, the control will be used.
|
25
29
|
@variant_resolver = lambda do |requested_variant|
|
26
|
-
requested_variant
|
30
|
+
requested_variant
|
27
31
|
end
|
28
32
|
|
29
33
|
# Tracking behavior can be implemented to link an event to an experiment.
|
@@ -31,8 +35,7 @@ module Gitlab
|
|
31
35
|
Configuration.logger.info "Gitlab::Experiment[#{name}] #{event}: #{args.merge(signature: signature)}"
|
32
36
|
end
|
33
37
|
|
34
|
-
# Called at the end of every experiment run, with the
|
35
|
-
# want to push the experiment into the client or push results elsewhere.
|
38
|
+
# Called at the end of every experiment run, with the result.
|
36
39
|
@publishing_behavior = lambda do |_result|
|
37
40
|
track(:assignment)
|
38
41
|
end
|
@@ -43,22 +46,17 @@ module Gitlab
|
|
43
46
|
Digest::MD5.hexdigest(([name] + hash_map.keys + values).join('|'))
|
44
47
|
end
|
45
48
|
|
46
|
-
# The domain for which this cookie applies so you can restrict to the domain level.
|
47
|
-
# When not set, it uses the current host. If you want to provide specific hosts, you can
|
48
|
-
# provide them either via an array like `['www.gitlab.com', .gitlab.com']`, or set it to `:all`.
|
49
|
-
@cookie_domain = :all
|
50
|
-
|
51
49
|
class << self
|
52
50
|
attr_accessor(
|
53
51
|
:name_prefix,
|
54
52
|
:logger,
|
55
53
|
:base_class,
|
56
54
|
:cache,
|
55
|
+
:cookie_domain,
|
57
56
|
:variant_resolver,
|
58
57
|
:tracking_behavior,
|
59
58
|
:publishing_behavior,
|
60
|
-
:context_hash_strategy
|
61
|
-
:cookie_domain
|
59
|
+
:context_hash_strategy
|
62
60
|
)
|
63
61
|
end
|
64
62
|
end
|
@@ -42,6 +42,14 @@ module Gitlab
|
|
42
42
|
@signature ||= { key: @experiment.key_for(@value), migration_keys: migration_keys }.compact
|
43
43
|
end
|
44
44
|
|
45
|
+
def method_missing(method_name, *)
|
46
|
+
@value.include?(method_name.to_sym) ? @value[method_name.to_sym] : super
|
47
|
+
end
|
48
|
+
|
49
|
+
def respond_to_missing?(method_name, *)
|
50
|
+
@value.include?(method_name.to_sym) ? true : super
|
51
|
+
end
|
52
|
+
|
45
53
|
private
|
46
54
|
|
47
55
|
def process_migrations(value)
|
@@ -11,7 +11,7 @@ module Gitlab
|
|
11
11
|
return hash if cookie_jar.nil?
|
12
12
|
|
13
13
|
resolver = [hash, :actor, cookie_name, cookie_jar.signed[cookie_name]]
|
14
|
-
resolve_cookie(*resolver)
|
14
|
+
resolve_cookie(*resolver) || generate_cookie(*resolver)
|
15
15
|
end
|
16
16
|
|
17
17
|
def cookie_jar
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
class Experiment
|
5
|
+
module RspecMatchers
|
6
|
+
extend RSpec::Matchers::DSL
|
7
|
+
|
8
|
+
matcher :exclude do |context|
|
9
|
+
ivar = :'@excluded'
|
10
|
+
|
11
|
+
match do |experiment|
|
12
|
+
experiment.context(context)
|
13
|
+
|
14
|
+
experiment.instance_variable_set(ivar, nil)
|
15
|
+
!experiment.run_callbacks(:exclusion_check) { :not_excluded }
|
16
|
+
end
|
17
|
+
|
18
|
+
failure_message do
|
19
|
+
%(expected #{context} to be excluded)
|
20
|
+
end
|
21
|
+
|
22
|
+
failure_message_when_negated do
|
23
|
+
%(expected #{context} not to be excluded)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
matcher :segment do |context|
|
28
|
+
ivar = :'@variant_name'
|
29
|
+
|
30
|
+
match do |experiment|
|
31
|
+
experiment.context(context)
|
32
|
+
|
33
|
+
experiment.instance_variable_set(ivar, nil)
|
34
|
+
experiment.run_callbacks(:segmented_run)
|
35
|
+
|
36
|
+
@actual = experiment.instance_variable_get(ivar)
|
37
|
+
@expected ? @actual.to_s == @expected.to_s : @actual.present?
|
38
|
+
end
|
39
|
+
|
40
|
+
chain :into do |expected|
|
41
|
+
raise ArgumentError, 'variant name must be provided' if expected.blank?
|
42
|
+
|
43
|
+
@expected = expected.to_s
|
44
|
+
end
|
45
|
+
|
46
|
+
failure_message do
|
47
|
+
%(expected #{context} to be segmented#{message_details})
|
48
|
+
end
|
49
|
+
|
50
|
+
failure_message_when_negated do
|
51
|
+
%(expected #{context} not to be segmented#{message_details})
|
52
|
+
end
|
53
|
+
|
54
|
+
def message_details
|
55
|
+
message = ''
|
56
|
+
message += %(\n into: #{@expected}) if @expected
|
57
|
+
message += %(\n actual: #{@actual}) if @actual
|
58
|
+
message
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
matcher :track do |event, *event_args|
|
63
|
+
match do |experiment|
|
64
|
+
wrapped(experiment) { |instance| expect(instance).to receive_with(event, *event_args) }
|
65
|
+
end
|
66
|
+
|
67
|
+
match_when_negated do |experiment|
|
68
|
+
wrapped(experiment) { |instance| expect(instance).not_to receive_with(event, *event_args) }
|
69
|
+
end
|
70
|
+
|
71
|
+
def wrapped(experiment, &block)
|
72
|
+
allow(experiment.class).to receive(:new).and_wrap_original do |new, *new_args|
|
73
|
+
new.call(*new_args).tap(&block)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def receive_with(*args)
|
78
|
+
receive(:track).with(*args) # rubocop:disable CodeReuse/ActiveRecord
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
RSpec.configure do |config|
|
86
|
+
config.define_derived_metadata(file_path: Regexp.new('/spec/experiments/')) do |metadata|
|
87
|
+
metadata[:type] = :experiment
|
88
|
+
end
|
89
|
+
|
90
|
+
config.include Gitlab::Experiment::RspecMatchers, :experiment
|
91
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-experiment
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -72,6 +72,7 @@ files:
|
|
72
72
|
- lib/gitlab/experiment/cookies.rb
|
73
73
|
- lib/gitlab/experiment/dsl.rb
|
74
74
|
- lib/gitlab/experiment/engine.rb
|
75
|
+
- lib/gitlab/experiment/rspec_matchers.rb
|
75
76
|
- lib/gitlab/experiment/variant.rb
|
76
77
|
- lib/gitlab/experiment/version.rb
|
77
78
|
homepage: https://gitlab.com/gitlab-org/gitlab-experiment
|