trailguide 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/views/layouts/trail_guide/admin/_footer.html.erb +1 -1
- data/lib/trail_guide/config.rb +4 -0
- data/lib/trail_guide/experiments/base.rb +12 -406
- data/lib/trail_guide/experiments/config.rb +4 -8
- data/lib/trail_guide/experiments/conversion.rb +63 -0
- data/lib/trail_guide/experiments/enrollment.rb +94 -0
- data/lib/trail_guide/experiments/lifecycle.rb +199 -0
- data/lib/trail_guide/experiments/persistence.rb +45 -0
- data/lib/trail_guide/experiments/results.rb +44 -0
- data/lib/trail_guide/variants.rb +39 -0
- data/lib/trail_guide/version.rb +1 -1
- data/lib/trailguide.rb +2 -1
- metadata +9 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad497aad0d4ac4af551fac70eb9191271586a4d3a3d9a6f45b9ae5879fb1cbaf
|
4
|
+
data.tar.gz: a2c20d6786785ee59e7c9bd15f14bb155078ff9f84ef4d1767aa939b3266cb00
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ebc9638bbdbbb2c9ad168441643fa61d12e9d1ecef0ca945fcd28157b2169ddf549b87f7e691c0735bd6c952be4552517f20cbb94a0002ea966e3351d845c02
|
7
|
+
data.tar.gz: 135fcf0f3bde5583232921bbb40c981ef29fbe044a178e41bf41891af9a5371fae403aa0de83000ed1b9ed4ba27bb886ea2a9e5e3d0ad06236cd52ec15c0d7d3
|
@@ -16,7 +16,7 @@
|
|
16
16
|
<span class="fas fa-flask" data-toggle="tooltip" title="experiments"></span>
|
17
17
|
</div>
|
18
18
|
<div class="col-sm-3 text-right">
|
19
|
-
<small class="text-muted"><%= TrailGuide.
|
19
|
+
<small class="text-muted"><%= TrailGuide.redis_client.id %><%= "/#{TrailGuide.redis.namespace}" if TrailGuide.redis.respond_to?(:namespace) %></small>
|
20
20
|
<button type="button" class="btn btn-link text-muted" style="font-size: 80%; margin: 0; padding: 0;" data-toggle="modal" data-target="#import-modal" data-tooltip="tooltip" title="Import/Export">
|
21
21
|
<span class="fas fa-file-download"></span>
|
22
22
|
</button>
|
data/lib/trail_guide/config.rb
CHANGED
@@ -1,9 +1,20 @@
|
|
1
1
|
require "trail_guide/experiments/config"
|
2
|
+
require "trail_guide/experiments/persistence"
|
3
|
+
require "trail_guide/experiments/lifecycle"
|
4
|
+
require "trail_guide/experiments/enrollment"
|
5
|
+
require "trail_guide/experiments/conversion"
|
6
|
+
require "trail_guide/experiments/results"
|
2
7
|
require "trail_guide/experiments/participant"
|
3
8
|
|
4
9
|
module TrailGuide
|
5
10
|
module Experiments
|
6
11
|
class Base
|
12
|
+
include Persistence
|
13
|
+
include Lifecycle
|
14
|
+
include Enrollment
|
15
|
+
include Conversion
|
16
|
+
include Results
|
17
|
+
|
7
18
|
class << self
|
8
19
|
delegate :groups, :algorithm, :control, :goals, :callbacks, :combined,
|
9
20
|
:combined?, :allow_multiple_conversions?, :allow_multiple_goals?,
|
@@ -23,10 +34,6 @@ module TrailGuide
|
|
23
34
|
configuration.configure(*args, &block)
|
24
35
|
end
|
25
36
|
|
26
|
-
def adapter
|
27
|
-
@adapter ||= TrailGuide::Adapters::Experiments::Redis.new(self)
|
28
|
-
end
|
29
|
-
|
30
37
|
# TODO alias name once specs have solid coverage
|
31
38
|
def experiment_name
|
32
39
|
configuration.name
|
@@ -48,173 +55,6 @@ module TrailGuide
|
|
48
55
|
combined.map { |combo| TrailGuide.catalog.find(combo) }
|
49
56
|
end
|
50
57
|
|
51
|
-
def run_callbacks(hook, *args)
|
52
|
-
return unless callbacks[hook]
|
53
|
-
return args[0] if hook == :rollout_winner # TODO do we need to account for this case here at the class level?
|
54
|
-
args.unshift(self)
|
55
|
-
callbacks[hook].each do |callback|
|
56
|
-
if callback.respond_to?(:call)
|
57
|
-
callback.call(*args)
|
58
|
-
else
|
59
|
-
send(callback, *args)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def start!(context=nil)
|
65
|
-
return false if started?
|
66
|
-
save! unless persisted?
|
67
|
-
started = adapter.set(:started_at, Time.now.to_i)
|
68
|
-
run_callbacks(:on_start, context)
|
69
|
-
started
|
70
|
-
end
|
71
|
-
|
72
|
-
def schedule!(start_at, stop_at=nil, context=nil)
|
73
|
-
return false if started?
|
74
|
-
save! unless persisted?
|
75
|
-
scheduled = adapter.set(:started_at, start_at.to_i)
|
76
|
-
adapter.set(:stopped_at, stop_at.to_i) if stop_at
|
77
|
-
run_callbacks(:on_schedule, start_at, stop_at, context)
|
78
|
-
scheduled
|
79
|
-
end
|
80
|
-
|
81
|
-
def pause!(context=nil)
|
82
|
-
return false unless running? && configuration.can_resume?
|
83
|
-
paused = adapter.set(:paused_at, Time.now.to_i)
|
84
|
-
run_callbacks(:on_pause, context)
|
85
|
-
paused
|
86
|
-
end
|
87
|
-
|
88
|
-
def stop!(context=nil)
|
89
|
-
return false unless started? && !stopped?
|
90
|
-
stopped = adapter.set(:stopped_at, Time.now.to_i)
|
91
|
-
run_callbacks(:on_stop, context)
|
92
|
-
stopped
|
93
|
-
end
|
94
|
-
|
95
|
-
def resume!(context=nil)
|
96
|
-
return false unless paused? && configuration.can_resume?
|
97
|
-
resumed = adapter.delete(:paused_at)
|
98
|
-
run_callbacks(:on_resume, context)
|
99
|
-
!!resumed
|
100
|
-
end
|
101
|
-
|
102
|
-
def started_at
|
103
|
-
started = adapter.get(:started_at)
|
104
|
-
return Time.at(started.to_i) if started
|
105
|
-
end
|
106
|
-
|
107
|
-
def paused_at
|
108
|
-
paused = adapter.get(:paused_at)
|
109
|
-
return Time.at(paused.to_i) if paused
|
110
|
-
end
|
111
|
-
|
112
|
-
def stopped_at
|
113
|
-
stopped = adapter.get(:stopped_at)
|
114
|
-
return Time.at(stopped.to_i) if stopped
|
115
|
-
end
|
116
|
-
|
117
|
-
def started?
|
118
|
-
time = started_at
|
119
|
-
time && time <= Time.now
|
120
|
-
end
|
121
|
-
|
122
|
-
def scheduled?
|
123
|
-
time = started_at
|
124
|
-
time && time > Time.now
|
125
|
-
end
|
126
|
-
|
127
|
-
def paused?
|
128
|
-
time = paused_at
|
129
|
-
time && time <= Time.now
|
130
|
-
end
|
131
|
-
|
132
|
-
def stopped?
|
133
|
-
time = stopped_at
|
134
|
-
time && time <= Time.now
|
135
|
-
end
|
136
|
-
|
137
|
-
def running?
|
138
|
-
started? && !paused? && !stopped?
|
139
|
-
end
|
140
|
-
|
141
|
-
def calibrating?
|
142
|
-
enable_calibration? && start_manually? && !started?
|
143
|
-
end
|
144
|
-
|
145
|
-
def fresh?
|
146
|
-
!started? && !scheduled? && !winner?
|
147
|
-
end
|
148
|
-
|
149
|
-
def declare_winner!(variant, context=nil)
|
150
|
-
variant = variants.find { |var| var == variant } unless variant.is_a?(Variant)
|
151
|
-
return false unless variant.present? && variant.experiment == self
|
152
|
-
run_callbacks(:on_winner, variant, context)
|
153
|
-
adapter.set(:winner, variant.name)
|
154
|
-
variant
|
155
|
-
end
|
156
|
-
|
157
|
-
def clear_winner!
|
158
|
-
adapter.delete(:winner)
|
159
|
-
end
|
160
|
-
|
161
|
-
def winner
|
162
|
-
winner = adapter.get(:winner)
|
163
|
-
return variants.find { |var| var == winner } if winner
|
164
|
-
end
|
165
|
-
|
166
|
-
def winner?
|
167
|
-
if combined?
|
168
|
-
combined.all? { |combo| TrailGuide.catalog.find(combo).winner? }
|
169
|
-
else
|
170
|
-
adapter.exists?(:winner)
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|
174
|
-
def persisted?
|
175
|
-
adapter.persisted?
|
176
|
-
end
|
177
|
-
|
178
|
-
def save!
|
179
|
-
combined_experiments.each(&:save!)
|
180
|
-
variants.each(&:save!)
|
181
|
-
adapter.setnx(:name, experiment_name)
|
182
|
-
end
|
183
|
-
|
184
|
-
def delete!(context=nil)
|
185
|
-
combined.each { |combo| TrailGuide.catalog.find(combo).delete! }
|
186
|
-
variants.each(&:delete!)
|
187
|
-
deleted = adapter.destroy
|
188
|
-
run_callbacks(:on_delete, context)
|
189
|
-
true
|
190
|
-
end
|
191
|
-
|
192
|
-
def reset!(context=nil)
|
193
|
-
delete!(context)
|
194
|
-
save!
|
195
|
-
run_callbacks(:on_reset, context)
|
196
|
-
true
|
197
|
-
end
|
198
|
-
|
199
|
-
def participants
|
200
|
-
variants.sum(&:participants)
|
201
|
-
end
|
202
|
-
|
203
|
-
def converted(checkpoint=nil)
|
204
|
-
variants.sum { |var| var.converted(checkpoint) }
|
205
|
-
end
|
206
|
-
|
207
|
-
def unconverted
|
208
|
-
participants - converted
|
209
|
-
end
|
210
|
-
|
211
|
-
def target_sample_size_reached?
|
212
|
-
return true unless configuration.target_sample_size
|
213
|
-
return true if participants >= configuration.target_sample_size
|
214
|
-
return false
|
215
|
-
end
|
216
|
-
|
217
|
-
# export the experiment state (not config) as json
|
218
58
|
def as_json(opts={})
|
219
59
|
{ experiment_name => {
|
220
60
|
started_at: started_at,
|
@@ -224,10 +64,6 @@ module TrailGuide
|
|
224
64
|
variants: variants.map(&:as_json).reduce({}) { |r,v| r.merge!(v) },
|
225
65
|
} }
|
226
66
|
end
|
227
|
-
|
228
|
-
def storage_key
|
229
|
-
configuration.name
|
230
|
-
end
|
231
67
|
end
|
232
68
|
|
233
69
|
attr_reader :participant
|
@@ -237,181 +73,7 @@ module TrailGuide
|
|
237
73
|
:enable_calibration?, :track_winner_conversions?, :callbacks, to: :class
|
238
74
|
|
239
75
|
def initialize(participant)
|
240
|
-
@participant =
|
241
|
-
end
|
242
|
-
|
243
|
-
def algorithm
|
244
|
-
@algorithm ||= self.class.algorithm.new(self)
|
245
|
-
end
|
246
|
-
|
247
|
-
def winning_variant
|
248
|
-
return nil unless winner?
|
249
|
-
run_callbacks(:rollout_winner, winner, participant)
|
250
|
-
end
|
251
|
-
|
252
|
-
def choose!(override: nil, metadata: nil, **opts)
|
253
|
-
return control if TrailGuide.configuration.disabled
|
254
|
-
|
255
|
-
variant = choose_variant!(override: override, metadata: metadata, **opts)
|
256
|
-
run_callbacks(:on_use, variant, participant, metadata)
|
257
|
-
variant
|
258
|
-
rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
|
259
|
-
run_callbacks(:on_redis_failover, e)
|
260
|
-
return variants.find { |var| var == override } || control if override.present?
|
261
|
-
return control
|
262
|
-
end
|
263
|
-
|
264
|
-
def choose_variant!(override: nil, excluded: false, metadata: nil)
|
265
|
-
return control if TrailGuide.configuration.disabled
|
266
|
-
|
267
|
-
if override.present?
|
268
|
-
variant = variants.find { |var| var == override } || control
|
269
|
-
if running? && !is_combined?
|
270
|
-
variant.increment_participation! if configuration.track_override
|
271
|
-
participant.participating!(variant) if configuration.store_override
|
272
|
-
end
|
273
|
-
return variant
|
274
|
-
end
|
275
|
-
|
276
|
-
if winner?
|
277
|
-
variant = winning_variant
|
278
|
-
if track_winner_conversions? && running?
|
279
|
-
variant.increment_participation! unless participant.variant == variant
|
280
|
-
participant.exit! if participant.participating? && participant.variant != variant
|
281
|
-
participant.participating!(variant)
|
282
|
-
end
|
283
|
-
return variant
|
284
|
-
end
|
285
|
-
|
286
|
-
return control if excluded || stopped?
|
287
|
-
|
288
|
-
if !started? && start_manually?
|
289
|
-
if enable_calibration?
|
290
|
-
unless participant.variant == control
|
291
|
-
control.increment_participation!
|
292
|
-
parent.control.increment_participation! if is_combined?
|
293
|
-
end
|
294
|
-
|
295
|
-
if participant.participating? && participant.variant != control
|
296
|
-
participant.exit!
|
297
|
-
parent.participant.exit! if is_combined?
|
298
|
-
end
|
299
|
-
|
300
|
-
participant.participating!(control)
|
301
|
-
parent.participant.participating!(parent.control) if is_combined?
|
302
|
-
end
|
303
|
-
return control
|
304
|
-
end
|
305
|
-
|
306
|
-
start! unless started? || scheduled?
|
307
|
-
return control unless running?
|
308
|
-
|
309
|
-
# only re-use the variant for experiments that store participation,
|
310
|
-
# all other (i.e. content-based) experiments should re-select and
|
311
|
-
# re-assign on enrollment
|
312
|
-
if configuration.sticky_assignment? && participant.participating?
|
313
|
-
variant = participant.variant
|
314
|
-
participant.participating!(variant)
|
315
|
-
return variant
|
316
|
-
end
|
317
|
-
|
318
|
-
return control unless is_combined? || TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
|
319
|
-
return control unless allow_participation?(metadata)
|
320
|
-
|
321
|
-
variant = algorithm_choose!(metadata: metadata)
|
322
|
-
variant.increment_participation!
|
323
|
-
participant.participating!(variant)
|
324
|
-
run_callbacks(:on_choose, variant, participant, metadata)
|
325
|
-
variant
|
326
|
-
end
|
327
|
-
|
328
|
-
def algorithm_choose!(metadata: nil)
|
329
|
-
algorithm.choose!(metadata: metadata)
|
330
|
-
end
|
331
|
-
|
332
|
-
def convert!(checkpoint=nil, metadata: nil)
|
333
|
-
if !started?
|
334
|
-
return false unless enable_calibration?
|
335
|
-
variant = participant.variant
|
336
|
-
return false unless variant.present? && variant == control
|
337
|
-
else
|
338
|
-
return false unless running?
|
339
|
-
variant = participant.variant
|
340
|
-
return false unless variant.present?
|
341
|
-
|
342
|
-
if winner?
|
343
|
-
return false unless track_winner_conversions? && variant == winner
|
344
|
-
end
|
345
|
-
end
|
346
|
-
|
347
|
-
if checkpoint.nil?
|
348
|
-
raise InvalidGoalError, "You must provide a valid goal checkpoint for `#{experiment_name}`." unless goals.empty?
|
349
|
-
else
|
350
|
-
goal = goals.find { |g| g == checkpoint }
|
351
|
-
raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." if goal.nil?
|
352
|
-
checkpoint = goal
|
353
|
-
end
|
354
|
-
|
355
|
-
# TODO eventually allow progressing through funnel checkpoints towards goals
|
356
|
-
if participant.converted?(checkpoint)
|
357
|
-
return false unless (checkpoint.nil? && allow_multiple_conversions?) || (checkpoint.present? && checkpoint.allow_multiple_conversions?)
|
358
|
-
elsif participant.converted?
|
359
|
-
return false unless allow_multiple_goals?
|
360
|
-
end
|
361
|
-
return false unless allow_conversion?(variant, checkpoint, metadata)
|
362
|
-
|
363
|
-
# TODO only reset if !reset_manually? AND they've converted all goals if
|
364
|
-
# allow_multiple_goals? is set
|
365
|
-
# TODO what should happen when allow_multiple_conversions? and !reset_manually?
|
366
|
-
# TODO eventually only reset if we're at the final goal in a funnel
|
367
|
-
participant.converted!(variant, checkpoint, reset: !reset_manually?)
|
368
|
-
variant.increment_conversion!(checkpoint)
|
369
|
-
if checkpoint.nil?
|
370
|
-
run_callbacks(:on_convert, checkpoint, variant, participant, metadata)
|
371
|
-
else
|
372
|
-
checkpoint.run_callbacks(:on_convert, self, variant, participant, metadata)
|
373
|
-
end
|
374
|
-
variant
|
375
|
-
rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
|
376
|
-
run_callbacks(:on_redis_failover, e)
|
377
|
-
return false
|
378
|
-
end
|
379
|
-
|
380
|
-
def allow_participation?(metadata=nil)
|
381
|
-
return true if callbacks[:allow_participation].empty?
|
382
|
-
run_callbacks(:allow_participation, true, participant, metadata)
|
383
|
-
end
|
384
|
-
|
385
|
-
def allow_conversion?(variant, checkpoint=nil, metadata=nil)
|
386
|
-
if checkpoint.nil?
|
387
|
-
return true if callbacks[:allow_conversion].empty?
|
388
|
-
# TODO why pass checkpoint through here if checkpoints are handled by their own method? it will always be nil here given current logic
|
389
|
-
run_callbacks(:allow_conversion, true, checkpoint, variant, participant, metadata)
|
390
|
-
else
|
391
|
-
checkpoint.allow_conversion?(self, variant, metadata)
|
392
|
-
end
|
393
|
-
end
|
394
|
-
|
395
|
-
def run_callbacks(hook, *args)
|
396
|
-
return unless callbacks[hook]
|
397
|
-
if [:allow_participation, :allow_conversion, :rollout_winner].include?(hook)
|
398
|
-
callbacks[hook].reduce(args.slice!(0,1)[0]) do |result, callback|
|
399
|
-
if callback.respond_to?(:call)
|
400
|
-
callback.call(self, result, *args)
|
401
|
-
else
|
402
|
-
send(callback, self, result, *args)
|
403
|
-
end
|
404
|
-
end
|
405
|
-
else
|
406
|
-
args.unshift(self)
|
407
|
-
callbacks[hook].each do |callback|
|
408
|
-
if callback.respond_to?(:call)
|
409
|
-
callback.call(*args)
|
410
|
-
else
|
411
|
-
send(callback, *args)
|
412
|
-
end
|
413
|
-
end
|
414
|
-
end
|
76
|
+
@participant = Experiments::Participant.new(self, participant)
|
415
77
|
end
|
416
78
|
|
417
79
|
def combined_experiments
|
@@ -419,62 +81,6 @@ module TrailGuide
|
|
419
81
|
combo.new(participant.participant)
|
420
82
|
end
|
421
83
|
end
|
422
|
-
|
423
|
-
### MEMOIZATOIN ###
|
424
|
-
# This is a lot of seemingly unnecessary duplication, but it really helps
|
425
|
-
# to cut down on the number of redis requests while still being
|
426
|
-
# thread-safe by memoizing these methods/values here at the instance level
|
427
|
-
|
428
|
-
def start!
|
429
|
-
@started_at = nil
|
430
|
-
self.class.start!
|
431
|
-
end
|
432
|
-
|
433
|
-
def started_at
|
434
|
-
@started_at ||= self.class.started_at
|
435
|
-
end
|
436
|
-
|
437
|
-
def paused_at
|
438
|
-
@paused_at ||= self.class.paused_at
|
439
|
-
end
|
440
|
-
|
441
|
-
def stopped_at
|
442
|
-
@stopped_at ||= self.class.stopped_at
|
443
|
-
end
|
444
|
-
|
445
|
-
def winner
|
446
|
-
@winner ||= self.class.winner
|
447
|
-
end
|
448
|
-
|
449
|
-
def scheduled?
|
450
|
-
started_at && started_at > Time.now
|
451
|
-
end
|
452
|
-
|
453
|
-
def started?
|
454
|
-
started_at && started_at <= Time.now
|
455
|
-
end
|
456
|
-
|
457
|
-
def paused?
|
458
|
-
paused_at && paused_at <= Time.now
|
459
|
-
end
|
460
|
-
|
461
|
-
def stopped?
|
462
|
-
stopped_at && stopped_at <= Time.now
|
463
|
-
end
|
464
|
-
|
465
|
-
def running?
|
466
|
-
started? && !paused? && !stopped?
|
467
|
-
end
|
468
|
-
|
469
|
-
def calibrating?
|
470
|
-
enable_calibration? && start_manually? && !started?
|
471
|
-
end
|
472
|
-
|
473
|
-
def winner?
|
474
|
-
return @has_winner unless @has_winner.nil?
|
475
|
-
@has_winner = self.class.winner?
|
476
|
-
end
|
477
|
-
|
478
84
|
end
|
479
85
|
end
|
480
86
|
end
|
@@ -19,11 +19,7 @@ module TrailGuide
|
|
19
19
|
def default_config
|
20
20
|
DEFAULT_KEYS.map do |key|
|
21
21
|
[key, nil]
|
22
|
-
end.to_h.merge(
|
23
|
-
variants: [],
|
24
|
-
goals: [],
|
25
|
-
combined: []
|
26
|
-
}).merge(callback_config)
|
22
|
+
end.to_h.merge(callback_config)
|
27
23
|
end
|
28
24
|
|
29
25
|
def callback_config
|
@@ -44,7 +40,7 @@ module TrailGuide
|
|
44
40
|
opts[:name] = nil
|
45
41
|
opts[:goals] = ancestor.goals.dup
|
46
42
|
opts[:combined] = ancestor.combined.dup
|
47
|
-
opts[:variants] = ancestor.variants.
|
43
|
+
opts[:variants] = ancestor.variants.dup(experiment)
|
48
44
|
opts = opts.merge(ancestor.callbacks.map { |k,v| [k,[v].flatten.compact] }.to_h)
|
49
45
|
end
|
50
46
|
super(*args, **opts, &block)
|
@@ -99,7 +95,7 @@ module TrailGuide
|
|
99
95
|
end
|
100
96
|
|
101
97
|
def variants
|
102
|
-
self[:variants]
|
98
|
+
self[:variants] ||= TrailGuide::Variants.new
|
103
99
|
end
|
104
100
|
|
105
101
|
def variant(varname, metadata: {}, weight: 1, control: false)
|
@@ -202,7 +198,7 @@ module TrailGuide
|
|
202
198
|
end
|
203
199
|
|
204
200
|
def combined
|
205
|
-
self[:combined]
|
201
|
+
self[:combined] ||= []
|
206
202
|
end
|
207
203
|
|
208
204
|
def combined?
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
module Experiments
|
3
|
+
module Conversion
|
4
|
+
def convert!(checkpoint=nil, metadata: nil)
|
5
|
+
if !started?
|
6
|
+
return false unless enable_calibration?
|
7
|
+
variant = participant.variant
|
8
|
+
return false unless variant.present? && variant == control
|
9
|
+
else
|
10
|
+
return false unless running?
|
11
|
+
variant = participant.variant
|
12
|
+
return false unless variant.present?
|
13
|
+
|
14
|
+
if winner?
|
15
|
+
return false unless track_winner_conversions? && variant == winner
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
if checkpoint.nil?
|
20
|
+
raise InvalidGoalError, "You must provide a valid goal checkpoint for `#{experiment_name}`." unless goals.empty?
|
21
|
+
else
|
22
|
+
goal = goals.find { |g| g == checkpoint }
|
23
|
+
raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." if goal.nil?
|
24
|
+
checkpoint = goal
|
25
|
+
end
|
26
|
+
|
27
|
+
# TODO eventually allow progressing through funnel checkpoints towards goals
|
28
|
+
if participant.converted?(checkpoint)
|
29
|
+
return false unless (checkpoint.nil? && allow_multiple_conversions?) || (checkpoint.present? && checkpoint.allow_multiple_conversions?)
|
30
|
+
elsif participant.converted?
|
31
|
+
return false unless allow_multiple_goals?
|
32
|
+
end
|
33
|
+
return false unless allow_conversion?(variant, checkpoint, metadata)
|
34
|
+
|
35
|
+
# TODO only reset if !reset_manually? AND they've converted all goals if
|
36
|
+
# allow_multiple_goals? is set
|
37
|
+
# TODO what should happen when allow_multiple_conversions? and !reset_manually?
|
38
|
+
# TODO eventually only reset if we're at the final goal in a funnel
|
39
|
+
participant.converted!(variant, checkpoint, reset: !reset_manually?)
|
40
|
+
variant.increment_conversion!(checkpoint)
|
41
|
+
if checkpoint.nil?
|
42
|
+
run_callbacks(:on_convert, checkpoint, variant, participant, metadata)
|
43
|
+
else
|
44
|
+
checkpoint.run_callbacks(:on_convert, self, variant, participant, metadata)
|
45
|
+
end
|
46
|
+
variant
|
47
|
+
rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
|
48
|
+
run_callbacks(:on_redis_failover, e)
|
49
|
+
return false
|
50
|
+
end
|
51
|
+
|
52
|
+
def allow_conversion?(variant, checkpoint=nil, metadata=nil)
|
53
|
+
if checkpoint.nil?
|
54
|
+
return true if callbacks[:allow_conversion].empty?
|
55
|
+
# TODO why pass checkpoint through here if checkpoints are handled by their own method? it will always be nil here given current logic
|
56
|
+
run_callbacks(:allow_conversion, true, checkpoint, variant, participant, metadata)
|
57
|
+
else
|
58
|
+
checkpoint.allow_conversion?(self, variant, metadata)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
module Experiments
|
3
|
+
module Enrollment
|
4
|
+
def algorithm
|
5
|
+
@algorithm ||= self.class.algorithm.new(self)
|
6
|
+
end
|
7
|
+
|
8
|
+
def choose!(override: nil, metadata: nil, **opts)
|
9
|
+
return control if TrailGuide.configuration.disabled
|
10
|
+
|
11
|
+
variant = choose_variant!(override: override, metadata: metadata, **opts)
|
12
|
+
run_callbacks(:on_use, variant, participant, metadata)
|
13
|
+
variant
|
14
|
+
rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
|
15
|
+
run_callbacks(:on_redis_failover, e)
|
16
|
+
return variants.find { |var| var == override } || control if override.present?
|
17
|
+
return control
|
18
|
+
end
|
19
|
+
|
20
|
+
def choose_variant!(override: nil, excluded: false, metadata: nil)
|
21
|
+
return control if TrailGuide.configuration.disabled
|
22
|
+
|
23
|
+
if override.present?
|
24
|
+
variant = variants.find { |var| var == override } || control
|
25
|
+
if running? && !is_combined?
|
26
|
+
variant.increment_participation! if configuration.track_override
|
27
|
+
participant.participating!(variant) if configuration.store_override
|
28
|
+
end
|
29
|
+
return variant
|
30
|
+
end
|
31
|
+
|
32
|
+
if winner?
|
33
|
+
variant = winning_variant
|
34
|
+
if track_winner_conversions? && running?
|
35
|
+
variant.increment_participation! unless participant.variant == variant
|
36
|
+
participant.exit! if participant.participating? && participant.variant != variant
|
37
|
+
participant.participating!(variant)
|
38
|
+
end
|
39
|
+
return variant
|
40
|
+
end
|
41
|
+
|
42
|
+
return control if excluded || stopped?
|
43
|
+
|
44
|
+
if !started? && start_manually?
|
45
|
+
if enable_calibration?
|
46
|
+
unless participant.variant == control
|
47
|
+
control.increment_participation!
|
48
|
+
parent.control.increment_participation! if is_combined?
|
49
|
+
end
|
50
|
+
|
51
|
+
if participant.participating? && participant.variant != control
|
52
|
+
participant.exit!
|
53
|
+
parent.participant.exit! if is_combined?
|
54
|
+
end
|
55
|
+
|
56
|
+
participant.participating!(control)
|
57
|
+
parent.participant.participating!(parent.control) if is_combined?
|
58
|
+
end
|
59
|
+
return control
|
60
|
+
end
|
61
|
+
|
62
|
+
start! unless started? || scheduled?
|
63
|
+
return control unless running?
|
64
|
+
|
65
|
+
# only re-use the variant for experiments that store participation,
|
66
|
+
# all other (i.e. content-based) experiments should re-select and
|
67
|
+
# re-assign on enrollment
|
68
|
+
if configuration.sticky_assignment? && participant.participating?
|
69
|
+
variant = participant.variant
|
70
|
+
participant.participating!(variant)
|
71
|
+
return variant
|
72
|
+
end
|
73
|
+
|
74
|
+
return control unless is_combined? || TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
|
75
|
+
return control unless allow_participation?(metadata)
|
76
|
+
|
77
|
+
variant = algorithm_choose!(metadata: metadata)
|
78
|
+
variant.increment_participation!
|
79
|
+
participant.participating!(variant)
|
80
|
+
run_callbacks(:on_choose, variant, participant, metadata)
|
81
|
+
variant
|
82
|
+
end
|
83
|
+
|
84
|
+
def algorithm_choose!(metadata: nil)
|
85
|
+
algorithm.choose!(metadata: metadata)
|
86
|
+
end
|
87
|
+
|
88
|
+
def allow_participation?(metadata=nil)
|
89
|
+
return true if callbacks[:allow_participation].empty?
|
90
|
+
run_callbacks(:allow_participation, true, participant, metadata)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,199 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
module Experiments
|
3
|
+
module Lifecycle
|
4
|
+
|
5
|
+
def self.included(base)
|
6
|
+
base.send :extend, ClassMethods
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def start!(context=nil)
|
11
|
+
return false if started?
|
12
|
+
save! unless persisted?
|
13
|
+
started = adapter.set(:started_at, Time.now.to_i)
|
14
|
+
run_callbacks(:on_start, context)
|
15
|
+
started
|
16
|
+
end
|
17
|
+
|
18
|
+
def schedule!(start_at, stop_at=nil, context=nil)
|
19
|
+
return false if started?
|
20
|
+
save! unless persisted?
|
21
|
+
scheduled = adapter.set(:started_at, start_at.to_i)
|
22
|
+
adapter.set(:stopped_at, stop_at.to_i) if stop_at
|
23
|
+
run_callbacks(:on_schedule, start_at, stop_at, context)
|
24
|
+
scheduled
|
25
|
+
end
|
26
|
+
|
27
|
+
def pause!(context=nil)
|
28
|
+
return false unless running? && configuration.can_resume?
|
29
|
+
paused = adapter.set(:paused_at, Time.now.to_i)
|
30
|
+
run_callbacks(:on_pause, context)
|
31
|
+
paused
|
32
|
+
end
|
33
|
+
|
34
|
+
def stop!(context=nil)
|
35
|
+
return false unless started? && !stopped?
|
36
|
+
stopped = adapter.set(:stopped_at, Time.now.to_i)
|
37
|
+
run_callbacks(:on_stop, context)
|
38
|
+
stopped
|
39
|
+
end
|
40
|
+
|
41
|
+
def resume!(context=nil)
|
42
|
+
return false unless paused? && configuration.can_resume?
|
43
|
+
resumed = adapter.delete(:paused_at)
|
44
|
+
run_callbacks(:on_resume, context)
|
45
|
+
!!resumed
|
46
|
+
end
|
47
|
+
|
48
|
+
def started_at
|
49
|
+
started = adapter.get(:started_at)
|
50
|
+
return Time.at(started.to_i) if started
|
51
|
+
end
|
52
|
+
|
53
|
+
def paused_at
|
54
|
+
paused = adapter.get(:paused_at)
|
55
|
+
return Time.at(paused.to_i) if paused
|
56
|
+
end
|
57
|
+
|
58
|
+
def stopped_at
|
59
|
+
stopped = adapter.get(:stopped_at)
|
60
|
+
return Time.at(stopped.to_i) if stopped
|
61
|
+
end
|
62
|
+
|
63
|
+
def started?
|
64
|
+
time = started_at
|
65
|
+
time && time <= Time.now
|
66
|
+
end
|
67
|
+
|
68
|
+
def scheduled?
|
69
|
+
time = started_at
|
70
|
+
time && time > Time.now
|
71
|
+
end
|
72
|
+
|
73
|
+
def paused?
|
74
|
+
time = paused_at
|
75
|
+
time && time <= Time.now
|
76
|
+
end
|
77
|
+
|
78
|
+
def stopped?
|
79
|
+
time = stopped_at
|
80
|
+
time && time <= Time.now
|
81
|
+
end
|
82
|
+
|
83
|
+
def running?
|
84
|
+
started? && !paused? && !stopped?
|
85
|
+
end
|
86
|
+
|
87
|
+
def calibrating?
|
88
|
+
enable_calibration? && start_manually? && !started?
|
89
|
+
end
|
90
|
+
|
91
|
+
def fresh?
|
92
|
+
!started? && !scheduled? && !winner?
|
93
|
+
end
|
94
|
+
|
95
|
+
def declare_winner!(variant, context=nil)
|
96
|
+
variant = variants.find { |var| var == variant } unless variant.is_a?(Variant)
|
97
|
+
return false unless variant.present? && variant.experiment == self
|
98
|
+
run_callbacks(:on_winner, variant, context)
|
99
|
+
adapter.set(:winner, variant.name)
|
100
|
+
variant
|
101
|
+
end
|
102
|
+
|
103
|
+
def clear_winner!
|
104
|
+
adapter.delete(:winner)
|
105
|
+
end
|
106
|
+
|
107
|
+
def winner?
|
108
|
+
if combined?
|
109
|
+
combined.all? { |combo| TrailGuide.catalog.find(combo).winner? }
|
110
|
+
else
|
111
|
+
adapter.exists?(:winner)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def run_callbacks(hook, *args)
|
116
|
+
return unless callbacks[hook]
|
117
|
+
return args[0] if hook == :rollout_winner # TODO do we need to account for this case here at the class level?
|
118
|
+
args.unshift(self)
|
119
|
+
callbacks[hook].each do |callback|
|
120
|
+
if callback.respond_to?(:call)
|
121
|
+
callback.call(*args)
|
122
|
+
else
|
123
|
+
send(callback, *args)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def run_callbacks(hook, *args)
|
130
|
+
return unless callbacks[hook]
|
131
|
+
if [:allow_participation, :allow_conversion, :rollout_winner].include?(hook)
|
132
|
+
callbacks[hook].reduce(args.slice!(0,1)[0]) do |result, callback|
|
133
|
+
if callback.respond_to?(:call)
|
134
|
+
callback.call(self, result, *args)
|
135
|
+
else
|
136
|
+
send(callback, self, result, *args)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
else
|
140
|
+
args.unshift(self)
|
141
|
+
callbacks[hook].each do |callback|
|
142
|
+
if callback.respond_to?(:call)
|
143
|
+
callback.call(*args)
|
144
|
+
else
|
145
|
+
send(callback, *args)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def start!
|
152
|
+
@started_at = nil
|
153
|
+
self.class.start!
|
154
|
+
end
|
155
|
+
|
156
|
+
def started_at
|
157
|
+
@started_at ||= self.class.started_at
|
158
|
+
end
|
159
|
+
|
160
|
+
def paused_at
|
161
|
+
@paused_at ||= self.class.paused_at
|
162
|
+
end
|
163
|
+
|
164
|
+
def stopped_at
|
165
|
+
@stopped_at ||= self.class.stopped_at
|
166
|
+
end
|
167
|
+
|
168
|
+
def scheduled?
|
169
|
+
started_at && started_at > Time.now
|
170
|
+
end
|
171
|
+
|
172
|
+
def started?
|
173
|
+
started_at && started_at <= Time.now
|
174
|
+
end
|
175
|
+
|
176
|
+
def paused?
|
177
|
+
paused_at && paused_at <= Time.now
|
178
|
+
end
|
179
|
+
|
180
|
+
def stopped?
|
181
|
+
stopped_at && stopped_at <= Time.now
|
182
|
+
end
|
183
|
+
|
184
|
+
def running?
|
185
|
+
started? && !paused? && !stopped?
|
186
|
+
end
|
187
|
+
|
188
|
+
def calibrating?
|
189
|
+
enable_calibration? && start_manually? && !started?
|
190
|
+
end
|
191
|
+
|
192
|
+
def winner?
|
193
|
+
return @has_winner unless @has_winner.nil?
|
194
|
+
@has_winner = self.class.winner?
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
module Experiments
|
3
|
+
module Persistence
|
4
|
+
|
5
|
+
def self.included(base)
|
6
|
+
base.send :extend, ClassMethods
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def adapter
|
11
|
+
@adapter ||= TrailGuide::Adapters::Experiments::Redis.new(self)
|
12
|
+
end
|
13
|
+
|
14
|
+
def persisted?
|
15
|
+
adapter.persisted?
|
16
|
+
end
|
17
|
+
|
18
|
+
def save!
|
19
|
+
combined_experiments.each(&:save!)
|
20
|
+
variants.each(&:save!)
|
21
|
+
adapter.setnx(:name, experiment_name)
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete!(context=nil)
|
25
|
+
combined.each { |combo| TrailGuide.catalog.find(combo).delete! }
|
26
|
+
variants.each(&:delete!)
|
27
|
+
deleted = adapter.destroy
|
28
|
+
run_callbacks(:on_delete, context)
|
29
|
+
true
|
30
|
+
end
|
31
|
+
|
32
|
+
def reset!(context=nil)
|
33
|
+
delete!(context)
|
34
|
+
save!
|
35
|
+
run_callbacks(:on_reset, context)
|
36
|
+
true
|
37
|
+
end
|
38
|
+
|
39
|
+
def storage_key
|
40
|
+
configuration.name
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
module Experiments
|
3
|
+
module Results
|
4
|
+
|
5
|
+
def self.included(base)
|
6
|
+
base.send :extend, ClassMethods
|
7
|
+
end
|
8
|
+
|
9
|
+
module ClassMethods
|
10
|
+
def winner
|
11
|
+
winner = adapter.get(:winner)
|
12
|
+
return variants.find { |var| var == winner } if winner
|
13
|
+
end
|
14
|
+
|
15
|
+
def participants
|
16
|
+
variants.sum(&:participants)
|
17
|
+
end
|
18
|
+
|
19
|
+
def converted(checkpoint=nil)
|
20
|
+
variants.sum { |var| var.converted(checkpoint) }
|
21
|
+
end
|
22
|
+
|
23
|
+
def unconverted
|
24
|
+
participants - converted
|
25
|
+
end
|
26
|
+
|
27
|
+
def target_sample_size_reached?
|
28
|
+
return true unless configuration.target_sample_size
|
29
|
+
return true if participants >= configuration.target_sample_size
|
30
|
+
return false
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def winner
|
35
|
+
@winner ||= self.class.winner
|
36
|
+
end
|
37
|
+
|
38
|
+
def winning_variant
|
39
|
+
return nil unless winner?
|
40
|
+
run_callbacks(:rollout_winner, winner, participant)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module TrailGuide
|
2
|
+
class Variants
|
3
|
+
attr_reader :variants
|
4
|
+
alias_method :to_a, :variants
|
5
|
+
delegate :each, :map, to: :variants
|
6
|
+
|
7
|
+
def initialize(*vars)
|
8
|
+
@variants = vars.flatten
|
9
|
+
end
|
10
|
+
|
11
|
+
def dup(experiment)
|
12
|
+
self.class.new(variants.map { |var| var.dup(experiment) })
|
13
|
+
end
|
14
|
+
|
15
|
+
def control
|
16
|
+
variants.find { |var| var.control? }
|
17
|
+
end
|
18
|
+
|
19
|
+
def method_missing(meth, *args, &block)
|
20
|
+
variant = variants.find { |var| var == meth }
|
21
|
+
return variant if variant.present?
|
22
|
+
|
23
|
+
if variants.respond_to?(meth, true)
|
24
|
+
result = variants.send(meth, *args, &block)
|
25
|
+
if result.is_a?(Array)
|
26
|
+
return self.class.new(result)
|
27
|
+
else
|
28
|
+
return result
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
super
|
33
|
+
end
|
34
|
+
|
35
|
+
def respond_to_missing?(meth, include_private=false)
|
36
|
+
variants.find { |var| var == meth }.present? || variants.respond_to?(meth, include_private)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/trail_guide/version.rb
CHANGED
data/lib/trailguide.rb
CHANGED
@@ -9,6 +9,7 @@ require "trail_guide/metrics"
|
|
9
9
|
require "trail_guide/calculators"
|
10
10
|
require "trail_guide/participant"
|
11
11
|
require "trail_guide/variant"
|
12
|
+
require "trail_guide/variants"
|
12
13
|
require "trail_guide/experiment"
|
13
14
|
require "trail_guide/combined_experiment"
|
14
15
|
require "trail_guide/catalog"
|
@@ -23,6 +24,6 @@ module TrailGuide
|
|
23
24
|
SCHEDULE_DATE_FORMAT = "%m/%d/%Y %I:%M %p %Z"
|
24
25
|
|
25
26
|
class << self
|
26
|
-
delegate :logger, :redis, to: :configuration
|
27
|
+
delegate :logger, :redis, :redis_client, to: :configuration
|
27
28
|
end
|
28
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: trailguide
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mark Rebec
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -269,7 +269,12 @@ files:
|
|
269
269
|
- lib/trail_guide/experiments/base.rb
|
270
270
|
- lib/trail_guide/experiments/combined_config.rb
|
271
271
|
- lib/trail_guide/experiments/config.rb
|
272
|
+
- lib/trail_guide/experiments/conversion.rb
|
273
|
+
- lib/trail_guide/experiments/enrollment.rb
|
274
|
+
- lib/trail_guide/experiments/lifecycle.rb
|
272
275
|
- lib/trail_guide/experiments/participant.rb
|
276
|
+
- lib/trail_guide/experiments/persistence.rb
|
277
|
+
- lib/trail_guide/experiments/results.rb
|
273
278
|
- lib/trail_guide/helper.rb
|
274
279
|
- lib/trail_guide/helper/experiment_proxy.rb
|
275
280
|
- lib/trail_guide/helper/helper_proxy.rb
|
@@ -282,6 +287,7 @@ files:
|
|
282
287
|
- lib/trail_guide/spec_helper.rb
|
283
288
|
- lib/trail_guide/unity.rb
|
284
289
|
- lib/trail_guide/variant.rb
|
290
|
+
- lib/trail_guide/variants.rb
|
285
291
|
- lib/trail_guide/version.rb
|
286
292
|
- lib/trailguide.rb
|
287
293
|
homepage: https://github.com/markrebec/trailguide
|
@@ -303,8 +309,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
303
309
|
- !ruby/object:Gem::Version
|
304
310
|
version: '0'
|
305
311
|
requirements: []
|
306
|
-
|
307
|
-
rubygems_version: 2.7.6
|
312
|
+
rubygems_version: 3.0.6
|
308
313
|
signing_key:
|
309
314
|
specification_version: 4
|
310
315
|
summary: User experiments for rails
|