trailguide 0.1.13 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,242 +1,9 @@
1
- require "trail_guide/experiment_config"
1
+ require "trail_guide/experiments/base"
2
2
 
3
3
  module TrailGuide
4
- class Experiment
5
- class << self
6
- delegate :metric, :algorithm, :control, :goals, :callbacks,
7
- :allow_multiple_conversions?, :allow_multiple_goals?, to: :configuration
8
- alias_method :funnels, :goals
9
-
10
- def inherited(child)
11
- TrailGuide::Catalog.register(child)
12
- end
13
-
14
- def configuration
15
- @configuration ||= ExperimentConfig.new(self)
16
- end
17
-
18
- def configure(*args, &block)
19
- configuration.configure(*args, &block)
20
- end
21
-
22
- def resettable?
23
- !configuration.reset_manually
24
- end
25
-
26
- def experiment_name
27
- configuration.name
28
- end
29
-
30
- def variants(include_control=true)
31
- if include_control
32
- configuration.variants
33
- else
34
- configuration.variants.select { |var| !var.control? }
35
- end
36
- end
37
-
38
- def run_callbacks(hook, *args)
39
- return unless callbacks[hook]
40
- args.unshift(self)
41
- callbacks[hook].each do |callback|
42
- if callback.respond_to?(:call)
43
- callback.call(*args)
44
- else
45
- send(callback, *args)
46
- end
47
- end
48
- end
49
-
50
- def start!
51
- return false if started?
52
- save! unless persisted?
53
- started = TrailGuide.redis.hset(storage_key, 'started_at', Time.now.to_i)
54
- run_callbacks(:on_start)
55
- started
56
- end
57
-
58
- def stop!
59
- return false unless running?
60
- stopped = TrailGuide.redis.hset(storage_key, 'stopped_at', Time.now.to_i)
61
- run_callbacks(:on_stop)
62
- stopped
63
- end
64
-
65
- def resume!
66
- return false unless started? && stopped?
67
- restarted = TrailGuide.redis.hdel(storage_key, 'stopped_at')
68
- run_callbacks(:on_resume)
69
- restarted
70
- end
71
-
72
- def started_at
73
- started = TrailGuide.redis.hget(storage_key, 'started_at')
74
- return Time.at(started.to_i) if started
75
- end
76
-
77
- def stopped_at
78
- stopped = TrailGuide.redis.hget(storage_key, 'stopped_at')
79
- return Time.at(stopped.to_i) if stopped
80
- end
81
-
82
- def started?
83
- !!started_at
84
- end
85
-
86
- def stopped?
87
- !!stopped_at
88
- end
89
-
90
- def running?
91
- started? && !stopped?
92
- end
93
-
94
- def declare_winner!(variant)
95
- variant = variant.name if variant.is_a?(Variant)
96
- TrailGuide.redis.hset(storage_key, 'winner', variant.to_s.underscore)
97
- end
98
-
99
- def clear_winner!
100
- TrailGuide.redis.hdel(storage_key, 'winner')
101
- end
102
-
103
- def winner
104
- winner = TrailGuide.redis.hget(storage_key, 'winner')
105
- return variants.find { |var| var == winner } if winner
106
- end
107
-
108
- def winner?
109
- TrailGuide.redis.hexists(storage_key, 'winner')
110
- end
111
-
112
- def persisted?
113
- TrailGuide.redis.exists(storage_key)
114
- end
115
-
116
- def save!
117
- variants.each(&:save!)
118
- TrailGuide.redis.hsetnx(storage_key, 'name', experiment_name)
119
- end
120
-
121
- def delete!
122
- variants.each(&:delete!)
123
- deleted = TrailGuide.redis.del(storage_key)
124
- run_callbacks(:on_delete)
125
- deleted
126
- end
127
-
128
- def reset!
129
- reset = (delete! && save!)
130
- run_callbacks(:on_reset)
131
- reset
132
- end
133
-
134
- def as_json(opts={})
135
- { experiment_name => {
136
- configuration: {
137
- metric: metric,
138
- algorithm: algorithm.name,
139
- variants: variants.as_json,
140
- goals: goals.as_json,
141
- resettable: resettable?,
142
- allow_multiple_conversions: allow_multiple_conversions?,
143
- allow_multiple_goals: allow_multiple_goals?
144
- },
145
- statistics: {
146
- # TODO expand on this for variants/goals
147
- participants: variants.sum(&:participants),
148
- converted: variants.sum(&:converted)
149
- }
150
- } }
151
- end
152
-
153
- def storage_key
154
- experiment_name
155
- end
156
- end
157
-
158
- attr_reader :participant
159
- delegate :configuration, :experiment_name, :variants, :control, :funnels,
160
- :storage_key, :running?, :started?, :started_at, :start!, :resettable?,
161
- :winner?, :winner, :allow_multiple_conversions?, :allow_multiple_goals?,
162
- :callbacks, to: :class
163
-
164
- def initialize(participant)
165
- @participant = participant
166
- end
167
-
168
- def algorithm
169
- @algorithm ||= self.class.algorithm.new(self)
170
- end
171
-
172
- def choose!(override: nil, metadata: nil, **opts)
173
- return control if TrailGuide.configuration.disabled
174
-
175
- variant = choose_variant!(override: override, metadata: metadata, **opts)
176
- participant.participating!(variant) unless override.present? && !configuration.store_override
177
- run_callbacks(:on_use, variant, metadata)
178
- variant
179
- end
180
-
181
- def choose_variant!(override: nil, excluded: false, metadata: nil)
182
- return control if TrailGuide.configuration.disabled
183
- if override.present?
184
- variant = variants.find { |var| var == override }
185
- return variant unless configuration.track_override && running?
186
- else
187
- return winner if winner?
188
- return control if excluded
189
- return control if !started? && configuration.start_manually
190
- start! unless started?
191
- return control unless running?
192
- return variants.find { |var| var == participant[storage_key] } if participating?
193
- return control unless TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
194
-
195
- variant = algorithm.choose!(metadata: metadata)
196
- end
197
-
198
- variant.increment_participation!
199
- run_callbacks(:on_choose, variant, metadata)
200
- variant
201
- end
202
-
203
- def convert!(checkpoint=nil, metadata: nil)
204
- return false unless participating?
205
- raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.present? || funnels.empty?
206
- raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.nil? || funnels.any? { |funnel| funnel == checkpoint.to_s.underscore.to_sym }
207
- # TODO eventually allow progressing through funnel checkpoints towards goals
208
- if converted?(checkpoint)
209
- return false unless allow_multiple_conversions?
210
- elsif converted?
211
- return false unless allow_multiple_goals?
212
- end
213
-
214
- variant = variants.find { |var| var == participant[storage_key] }
215
- # TODO eventually only reset if we're at the final goal in a funnel
216
- participant.converted!(variant, checkpoint, reset: resettable?)
217
- variant.increment_conversion!(checkpoint)
218
- run_callbacks(:on_convert, variant, checkpoint, metadata)
219
- variant
220
- end
221
-
222
- def participating?
223
- participant.participating?(self)
224
- end
225
-
226
- def converted?(checkpoint=nil)
227
- participant.converted?(self, checkpoint)
228
- end
229
-
230
- def run_callbacks(hook, *args)
231
- return unless callbacks[hook]
232
- args.unshift(self)
233
- callbacks[hook].each do |callback|
234
- if callback.respond_to?(:call)
235
- callback.call(*args)
236
- else
237
- send(callback, *args)
238
- end
239
- end
4
+ class Experiment < Experiments::Base
5
+ def self.inherited(child)
6
+ TrailGuide.catalog.register(child)
240
7
  end
241
8
  end
242
9
  end
@@ -0,0 +1,268 @@
1
+ require "trail_guide/experiments/config"
2
+
3
+ module TrailGuide
4
+ module Experiments
5
+ class Base
6
+ class << self
7
+ delegate :metric, :algorithm, :control, :goals, :callbacks, :combined,
8
+ :combined?, :allow_multiple_conversions?, :allow_multiple_goals?,
9
+ :track_winner_conversions?, to: :configuration
10
+ alias_method :funnels, :goals
11
+
12
+ def configuration
13
+ @configuration ||= Experiments::Config.new(self)
14
+ end
15
+
16
+ def configure(*args, &block)
17
+ configuration.configure(*args, &block)
18
+ end
19
+
20
+ def resettable?
21
+ !configuration.reset_manually
22
+ end
23
+
24
+ def experiment_name
25
+ configuration.name
26
+ end
27
+
28
+ def variants(include_control=true)
29
+ if include_control
30
+ configuration.variants
31
+ else
32
+ configuration.variants.select { |var| !var.control? }
33
+ end
34
+ end
35
+
36
+ def run_callbacks(hook, *args)
37
+ return unless callbacks[hook]
38
+ return args[0] if hook == :return_winner
39
+ args.unshift(self)
40
+ callbacks[hook].each do |callback|
41
+ if callback.respond_to?(:call)
42
+ callback.call(*args)
43
+ else
44
+ send(callback, *args)
45
+ end
46
+ end
47
+ end
48
+
49
+ def start!
50
+ return false if started?
51
+ save! unless persisted?
52
+ started = TrailGuide.redis.hset(storage_key, 'started_at', Time.now.to_i)
53
+ run_callbacks(:on_start)
54
+ started
55
+ end
56
+
57
+ def stop!
58
+ return false unless running?
59
+ stopped = TrailGuide.redis.hset(storage_key, 'stopped_at', Time.now.to_i)
60
+ run_callbacks(:on_stop)
61
+ stopped
62
+ end
63
+
64
+ def resume!
65
+ return false unless started? && stopped?
66
+ restarted = TrailGuide.redis.hdel(storage_key, 'stopped_at')
67
+ run_callbacks(:on_resume)
68
+ restarted
69
+ end
70
+
71
+ def started_at
72
+ started = TrailGuide.redis.hget(storage_key, 'started_at')
73
+ return Time.at(started.to_i) if started
74
+ end
75
+
76
+ def stopped_at
77
+ stopped = TrailGuide.redis.hget(storage_key, 'stopped_at')
78
+ return Time.at(stopped.to_i) if stopped
79
+ end
80
+
81
+ def started?
82
+ !!started_at
83
+ end
84
+
85
+ def stopped?
86
+ !!stopped_at
87
+ end
88
+
89
+ def running?
90
+ started? && !stopped?
91
+ end
92
+
93
+ def declare_winner!(variant)
94
+ variant = variants.find { |var| var == variant } unless variant.is_a?(Variant)
95
+ run_callbacks(:on_winner, variant)
96
+ TrailGuide.redis.hset(storage_key, 'winner', variant.name.to_s.underscore)
97
+ end
98
+
99
+ def clear_winner!
100
+ TrailGuide.redis.hdel(storage_key, 'winner')
101
+ end
102
+
103
+ def winner
104
+ winner = TrailGuide.redis.hget(storage_key, 'winner')
105
+ return variants.find { |var| var == winner } if winner
106
+ end
107
+
108
+ def winner?
109
+ TrailGuide.redis.hexists(storage_key, 'winner')
110
+ end
111
+
112
+ def persisted?
113
+ TrailGuide.redis.exists(storage_key)
114
+ end
115
+
116
+ def save!
117
+ combined.each { |combo| TrailGuide.catalog.find(combo).save! }
118
+ variants.each(&:save!)
119
+ TrailGuide.redis.hsetnx(storage_key, 'name', experiment_name)
120
+ end
121
+
122
+ def delete!
123
+ combined.each { |combo| TrailGuide.catalog.find(combo).delete! }
124
+ variants.each(&:delete!)
125
+ deleted = TrailGuide.redis.del(storage_key)
126
+ run_callbacks(:on_delete)
127
+ deleted
128
+ end
129
+
130
+ def reset!
131
+ reset = (delete! && save!)
132
+ run_callbacks(:on_reset)
133
+ reset
134
+ end
135
+
136
+ def as_json(opts={})
137
+ { experiment_name => {
138
+ configuration: {
139
+ metric: metric,
140
+ algorithm: algorithm.name,
141
+ variants: variants.as_json,
142
+ goals: goals.as_json,
143
+ resettable: resettable?,
144
+ allow_multiple_conversions: allow_multiple_conversions?,
145
+ allow_multiple_goals: allow_multiple_goals?
146
+ },
147
+ statistics: {
148
+ # TODO expand on this for variants/goals
149
+ participants: variants.sum(&:participants),
150
+ converted: variants.sum(&:converted)
151
+ }
152
+ } }
153
+ end
154
+
155
+ def storage_key
156
+ configuration.name
157
+ end
158
+ end
159
+
160
+ attr_reader :participant
161
+ delegate :configuration, :experiment_name, :variants, :control, :funnels,
162
+ :storage_key, :running?, :started?, :started_at, :start!, :resettable?,
163
+ :winner?, :allow_multiple_conversions?, :allow_multiple_goals?,
164
+ :track_winner_conversions?, :callbacks, to: :class
165
+
166
+ def initialize(participant)
167
+ @participant = participant
168
+ end
169
+
170
+ def algorithm
171
+ @algorithm ||= self.class.algorithm.new(self)
172
+ end
173
+
174
+ def winner
175
+ run_callbacks(:return_winner, self.class.winner)
176
+ end
177
+
178
+ def choose!(override: nil, metadata: nil, **opts)
179
+ return control if TrailGuide.configuration.disabled
180
+
181
+ variant = choose_variant!(override: override, metadata: metadata, **opts)
182
+ participant.participating!(variant) unless override.present? && !configuration.store_override
183
+ run_callbacks(:on_use, variant, metadata)
184
+ variant
185
+ end
186
+
187
+ def choose_variant!(override: nil, excluded: false, metadata: nil)
188
+ return control if TrailGuide.configuration.disabled
189
+ if override.present?
190
+ variant = variants.find { |var| var == override }
191
+ return variant unless configuration.track_override && running?
192
+ else
193
+ if winner?
194
+ variant = winner
195
+ return variant unless track_winner_conversions? && running?
196
+ else
197
+ return control if excluded
198
+ return control if !started? && configuration.start_manually
199
+ start! unless started?
200
+ return control unless running?
201
+ return variants.find { |var| var == participant[storage_key] } if participating?
202
+ return control unless TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
203
+
204
+ variant = algorithm_choose!(metadata: metadata)
205
+ end
206
+ end
207
+
208
+ variant.increment_participation!
209
+ run_callbacks(:on_choose, variant, metadata)
210
+ variant
211
+ end
212
+
213
+ def algorithm_choose!(metadata: nil)
214
+ algorithm.choose!(metadata: metadata)
215
+ end
216
+
217
+ def convert!(checkpoint=nil, metadata: nil)
218
+ return false if !running? || (winner? && !track_winner_conversions?)
219
+ return false unless participating?
220
+ raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.present? || funnels.empty?
221
+ raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.nil? || funnels.any? { |funnel| funnel == checkpoint.to_s.underscore.to_sym }
222
+ # TODO eventually allow progressing through funnel checkpoints towards goals
223
+ if converted?(checkpoint)
224
+ return false unless allow_multiple_conversions?
225
+ elsif converted?
226
+ return false unless allow_multiple_goals?
227
+ end
228
+
229
+ variant = variants.find { |var| var == participant[storage_key] }
230
+ # TODO eventually only reset if we're at the final goal in a funnel
231
+ participant.converted!(variant, checkpoint, reset: resettable?)
232
+ variant.increment_conversion!(checkpoint)
233
+ run_callbacks(:on_convert, variant, checkpoint, metadata)
234
+ variant
235
+ end
236
+
237
+ def participating?
238
+ participant.participating?(self)
239
+ end
240
+
241
+ def converted?(checkpoint=nil)
242
+ participant.converted?(self, checkpoint)
243
+ end
244
+
245
+ def run_callbacks(hook, *args)
246
+ return unless callbacks[hook]
247
+ if hook == :return_winner
248
+ callbacks[hook].reduce(args[0]) do |winner, callback|
249
+ if callback.respond_to?(:call)
250
+ callback.call(self, winner)
251
+ else
252
+ send(callback, self, winner)
253
+ end
254
+ end
255
+ else
256
+ args.unshift(self)
257
+ callbacks[hook].each do |callback|
258
+ if callback.respond_to?(:call)
259
+ callback.call(*args)
260
+ else
261
+ send(callback, *args)
262
+ end
263
+ end
264
+ end
265
+ end
266
+ end
267
+ end
268
+ end
@@ -0,0 +1,12 @@
1
+ require "trail_guide/experiments/config"
2
+
3
+ module TrailGuide
4
+ module Experiments
5
+ class CombinedConfig < Config
6
+ def initialize(experiment, *args, **opts, &block)
7
+ args.push(:parent)
8
+ super(experiment, *args, **opts, &block)
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,158 @@
1
+ module TrailGuide
2
+ module Experiments
3
+ class Config < Canfig::Config
4
+ ENGINE_CONFIG_KEYS = [
5
+ :start_manually, :reset_manually, :store_override, :track_override,
6
+ :algorithm, :allow_multiple_conversions, :allow_multiple_goals,
7
+ :track_winner_conversions
8
+ ].freeze
9
+
10
+ def self.engine_config
11
+ ENGINE_CONFIG_KEYS.map do |key|
12
+ [key, TrailGuide.configuration.send(key.to_sym)]
13
+ end.to_h
14
+ end
15
+
16
+ def self.default_config
17
+ { name: nil, metric: nil, variants: [], goals: [], combined: [] }
18
+ end
19
+
20
+ def self.callbacks_config
21
+ {
22
+ callbacks: {
23
+ on_choose: [TrailGuide.configuration.on_experiment_choose].flatten.compact,
24
+ on_use: [TrailGuide.configuration.on_experiment_use].flatten.compact,
25
+ on_convert: [TrailGuide.configuration.on_experiment_convert].flatten.compact,
26
+ on_start: [TrailGuide.configuration.on_experiment_start].flatten.compact,
27
+ on_stop: [TrailGuide.configuration.on_experiment_stop].flatten.compact,
28
+ on_resume: [TrailGuide.configuration.on_experiment_resume].flatten.compact,
29
+ on_winner: [TrailGuide.configuration.on_experiment_winner].flatten.compact,
30
+ on_reset: [TrailGuide.configuration.on_experiment_reset].flatten.compact,
31
+ on_delete: [TrailGuide.configuration.on_experiment_delete].flatten.compact,
32
+ return_winner: [TrailGuide.configuration.return_experiment_winner].flatten.compact,
33
+ }
34
+ }
35
+ end
36
+
37
+ attr_reader :experiment
38
+
39
+ def initialize(experiment, *args, **opts, &block)
40
+ @experiment = experiment
41
+ opts = opts.merge(self.class.engine_config)
42
+ opts = opts.merge(self.class.default_config)
43
+ opts = opts.merge(self.class.callbacks_config)
44
+ super(*args, **opts, &block)
45
+ end
46
+
47
+ def resettable?
48
+ !reset_manually
49
+ end
50
+
51
+ def allow_multiple_conversions?
52
+ allow_multiple_conversions
53
+ end
54
+
55
+ def allow_multiple_goals?
56
+ allow_multiple_goals
57
+ end
58
+
59
+ def track_winner_conversions?
60
+ track_winner_conversions
61
+ end
62
+
63
+ def name
64
+ @name ||= (self[:name] || experiment.name).try(:to_s).try(:underscore).try(:to_sym)
65
+ end
66
+
67
+ def metric
68
+ @metric ||= (self[:metric] || name).try(:to_s).try(:underscore).try(:to_sym)
69
+ end
70
+
71
+ def algorithm
72
+ @algorithm ||= TrailGuide::Algorithms.algorithm(self[:algorithm])
73
+ end
74
+
75
+ def variant(varname, metadata: {}, weight: 1, control: false)
76
+ raise ArgumentError, "The variant `#{varname}` already exists in the experiment `#{name}`" if variants.any? { |var| var == varname }
77
+ control = true if variants.empty?
78
+ variants.each(&:variant!) if control
79
+ variant = Variant.new(experiment, varname, metadata: metadata, weight: weight, control: control)
80
+ variants << variant
81
+ variant
82
+ end
83
+
84
+ def control
85
+ return variants.find { |var| var.control? } || variants.first
86
+ end
87
+
88
+ def control=(name)
89
+ variants.each(&:variant!)
90
+ var_idx = variants.index { |var| var == name }
91
+
92
+ if var_idx.nil?
93
+ variant = Variant.new(experiment, name, control: true)
94
+ variants.push(variant)
95
+ else
96
+ variant = variants[var_idx]
97
+ variant.control!
98
+ end
99
+
100
+ variant
101
+ end
102
+
103
+ def goal(name)
104
+ goals << name.to_s.underscore.to_sym
105
+ end
106
+ alias_method :funnel, :goal
107
+
108
+ def goals
109
+ self[:goals]
110
+ end
111
+ alias_method :funnels, :goals
112
+
113
+ def combined?
114
+ !combined.empty?
115
+ end
116
+
117
+ def on_choose(meth=nil, &block)
118
+ callbacks[:on_choose] << (meth || block)
119
+ end
120
+
121
+ def on_use(meth=nil, &block)
122
+ callbacks[:on_use] << (meth || block)
123
+ end
124
+
125
+ def on_convert(meth=nil, &block)
126
+ callbacks[:on_convert] << (meth || block)
127
+ end
128
+
129
+ def on_start(meth=nil, &block)
130
+ callbacks[:on_start] << (meth || block)
131
+ end
132
+
133
+ def on_stop(meth=nil, &block)
134
+ callbacks[:on_stop] << (meth || block)
135
+ end
136
+
137
+ def on_resume(meth=nil, &block)
138
+ callbacks[:on_resume] << (meth || block)
139
+ end
140
+
141
+ def on_winner(meth=nil, &block)
142
+ callbacks[:on_winner] << (meth || block)
143
+ end
144
+
145
+ def on_reset(meth=nil, &block)
146
+ callbacks[:on_reset] << (meth || block)
147
+ end
148
+
149
+ def on_delete(meth=nil, &block)
150
+ callbacks[:on_delete] << (meth || block)
151
+ end
152
+
153
+ def return_winner(meth=nil, &block)
154
+ callbacks[:return_winner] << (meth || block)
155
+ end
156
+ end
157
+ end
158
+ end
@@ -121,7 +121,7 @@ module TrailGuide
121
121
  end
122
122
 
123
123
  def experiments
124
- @experiments ||= TrailGuide::Catalog.select(metric).map do |experiment|
124
+ @experiments ||= TrailGuide.catalog.select(metric).map do |experiment|
125
125
  experiment.new(participant)
126
126
  end
127
127
  end