trailguide 0.1.31 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +73 -8
  3. data/app/assets/javascripts/trail_guide/admin/application.js +20 -1
  4. data/app/assets/stylesheets/trail_guide/admin/application.css +22 -0
  5. data/app/assets/stylesheets/trail_guide/admin/experiments.css +36 -3
  6. data/app/controllers/trail_guide/admin/application_controller.rb +59 -8
  7. data/app/controllers/trail_guide/admin/experiments_controller.rb +209 -16
  8. data/app/controllers/trail_guide/admin/groups_controller.rb +34 -0
  9. data/app/controllers/trail_guide/experiments_controller.rb +1 -1
  10. data/app/views/layouts/trail_guide/admin/_calculator.erb +24 -0
  11. data/app/views/layouts/trail_guide/admin/_footer.html.erb +27 -0
  12. data/app/views/layouts/trail_guide/admin/_header.html.erb +147 -0
  13. data/app/views/layouts/trail_guide/admin/_import_modal.html.erb +45 -0
  14. data/app/views/layouts/trail_guide/admin/application.html.erb +17 -3
  15. data/app/views/trail_guide/admin/experiments/_alert_peek.html.erb +19 -0
  16. data/app/views/trail_guide/admin/experiments/_alert_state.html.erb +49 -0
  17. data/app/views/trail_guide/admin/experiments/_btn_analyze.html.erb +11 -0
  18. data/app/views/trail_guide/admin/experiments/_btn_analyze_goal.html.erb +5 -0
  19. data/app/views/trail_guide/admin/experiments/_btn_convert.html.erb +33 -0
  20. data/app/views/trail_guide/admin/experiments/_btn_enroll.html.erb +3 -0
  21. data/app/views/trail_guide/admin/experiments/_btn_join.html.erb +11 -0
  22. data/app/views/trail_guide/admin/experiments/_btn_leave.html.erb +7 -0
  23. data/app/views/trail_guide/admin/experiments/_btn_pause.html.erb +5 -0
  24. data/app/views/trail_guide/admin/experiments/_btn_peek.html.erb +13 -0
  25. data/app/views/trail_guide/admin/experiments/_btn_reset.html.erb +5 -0
  26. data/app/views/trail_guide/admin/experiments/_btn_restart.html.erb +5 -0
  27. data/app/views/trail_guide/admin/experiments/_btn_resume.html.erb +5 -0
  28. data/app/views/trail_guide/admin/experiments/_btn_schedule.html.erb +6 -0
  29. data/app/views/trail_guide/admin/experiments/_btn_start.html.erb +5 -0
  30. data/app/views/trail_guide/admin/experiments/_btn_stop.html.erb +5 -0
  31. data/app/views/trail_guide/admin/experiments/_experiment.html.erb +68 -172
  32. data/app/views/trail_guide/admin/experiments/_header.html.erb +87 -0
  33. data/app/views/trail_guide/admin/experiments/_start_modal.html.erb +57 -0
  34. data/app/views/trail_guide/admin/experiments/_tbody.html.erb +112 -0
  35. data/app/views/trail_guide/admin/experiments/_thead.html.erb +38 -0
  36. data/app/views/trail_guide/admin/experiments/index.html.erb +8 -16
  37. data/app/views/trail_guide/admin/experiments/show.html.erb +78 -0
  38. data/app/views/trail_guide/admin/groups/index.html.erb +40 -0
  39. data/app/views/trail_guide/admin/groups/show.html.erb +9 -0
  40. data/app/views/trail_guide/admin/orphans/_alert.html.erb +44 -0
  41. data/config/initializers/trailguide.rb +72 -9
  42. data/config/routes/admin.rb +19 -12
  43. data/lib/trail_guide/admin/engine.rb +2 -0
  44. data/lib/trail_guide/admin.rb +2 -0
  45. data/lib/trail_guide/calculators/bayesian.rb +98 -0
  46. data/lib/trail_guide/calculators/calculator.rb +58 -0
  47. data/lib/trail_guide/calculators/score.rb +68 -0
  48. data/lib/trail_guide/calculators.rb +8 -0
  49. data/lib/trail_guide/catalog.rb +134 -19
  50. data/lib/trail_guide/combined_experiment.rb +8 -3
  51. data/lib/trail_guide/config.rb +6 -1
  52. data/lib/trail_guide/engine.rb +2 -0
  53. data/lib/trail_guide/errors.rb +30 -1
  54. data/lib/trail_guide/experiment.rb +0 -1
  55. data/lib/trail_guide/experiments/base.rb +189 -53
  56. data/lib/trail_guide/experiments/config.rb +82 -13
  57. data/lib/trail_guide/experiments/participant.rb +21 -1
  58. data/lib/trail_guide/helper.rb +59 -32
  59. data/lib/trail_guide/metrics/checkpoint.rb +24 -0
  60. data/lib/trail_guide/metrics/config.rb +45 -0
  61. data/lib/trail_guide/metrics/funnel.rb +24 -0
  62. data/lib/trail_guide/metrics/goal.rb +89 -0
  63. data/lib/trail_guide/metrics.rb +9 -0
  64. data/lib/trail_guide/participant.rb +54 -21
  65. data/lib/trail_guide/variant.rb +34 -12
  66. data/lib/trail_guide/version.rb +2 -2
  67. data/lib/trailguide.rb +4 -0
  68. metadata +112 -7
  69. data/app/views/layouts/trail_guide/admin/_footer.erb +0 -10
  70. data/app/views/layouts/trail_guide/admin/_header.erb +0 -42
  71. data/app/views/trail_guide/admin/experiments/_combined_experiment.html.erb +0 -189
  72. data/config/routes.rb +0 -5
@@ -1,10 +1,10 @@
1
1
  module TrailGuide
2
2
  module Helper
3
- def trailguide(metric=nil, **opts, &block)
3
+ def trailguide(key=nil, **opts, &block)
4
4
  @trailguide_proxy ||= HelperProxy.new(self)
5
5
  @trailguide_proxy = HelperProxy.new(self) if @trailguide_proxy.context != self
6
- return @trailguide_proxy if metric.nil?
7
- @trailguide_proxy.choose!(metric, **opts, &block)
6
+ return @trailguide_proxy if key.nil?
7
+ @trailguide_proxy.choose!(key, **opts, &block)
8
8
  end
9
9
 
10
10
  def trailguide_participant
@@ -70,42 +70,42 @@ module TrailGuide
70
70
  @participant = participant
71
71
  end
72
72
 
73
- def new(metric)
74
- MetricProxy.new(context, metric, participant: participant)
73
+ def new(key)
74
+ ExperimentProxy.new(context, key, participant: participant)
75
75
  end
76
76
 
77
- def choose!(metric, **opts, &block)
78
- new(metric).choose!(**opts, &block)
77
+ def choose!(key, **opts, &block)
78
+ new(key).choose!(**opts, &block)
79
79
  end
80
80
  alias_method :enroll!, :choose!
81
81
 
82
- def choose(metric, **opts, &block)
83
- new(metric).choose(**opts, &block)
82
+ def choose(key, **opts, &block)
83
+ new(key).choose(**opts, &block)
84
84
  end
85
85
  alias_method :enroll, :choose
86
86
 
87
- def run!(metric, **opts)
88
- new(metric).run!(**opts)
87
+ def run!(key, **opts)
88
+ new(key).run!(**opts)
89
89
  end
90
90
 
91
- def run(metric, **opts)
92
- new(metric).run(**opts)
91
+ def run(key, **opts)
92
+ new(key).run(**opts)
93
93
  end
94
94
 
95
- def render!(metric, **opts)
96
- new(metric).render!(**opts)
95
+ def render!(key, **opts)
96
+ new(key).render!(**opts)
97
97
  end
98
98
 
99
- def render(metric, **opts)
100
- new(metric).render(**opts)
99
+ def render(key, **opts)
100
+ new(key).render(**opts)
101
101
  end
102
102
 
103
- def convert!(metric, checkpoint=nil, **opts, &block)
104
- new(metric).convert!(checkpoint, **opts, &block)
103
+ def convert!(key, checkpoint=nil, **opts, &block)
104
+ new(key).convert!(checkpoint, **opts, &block)
105
105
  end
106
106
 
107
- def convert(metric, checkpoint=nil, **opts, &block)
108
- new(metric).convert(checkpoint, **opts, &block)
107
+ def convert(key, checkpoint=nil, **opts, &block)
108
+ new(key).convert(checkpoint, **opts, &block)
109
109
  end
110
110
 
111
111
  def participant
@@ -121,17 +121,18 @@ module TrailGuide
121
121
  end
122
122
  end
123
123
 
124
- class MetricProxy < HelperProxy
125
- attr_reader :metric
124
+ class ExperimentProxy < HelperProxy
125
+ attr_reader :key
126
126
 
127
- def initialize(context, metric, **opts)
127
+ def initialize(context, key, **opts)
128
128
  super(context, **opts)
129
- @metric = metric
130
- raise NoExperimentsError, "Could not find any experiments matching `#{metric}`." if experiments.empty?
129
+ @key = key.to_s.underscore.to_sym
131
130
  end
132
131
 
133
132
  def choose!(**opts, &block)
134
- raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but the metric `#{metric}` matches more than one experiment." if experiments.length > 1
133
+ raise NoExperimentsError, key if experiments.empty?
134
+ raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but `#{key}` matches more than one experiment." if experiments.length > 1
135
+ raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but `#{key}` refers to a combined experiment." if experiment.combined?
135
136
  opts = {override: override_variant, excluded: exclude_visitor?}.merge(opts)
136
137
  variant = experiment.choose!(**opts)
137
138
  if block_given?
@@ -144,6 +145,8 @@ module TrailGuide
144
145
 
145
146
  def choose(**opts, &block)
146
147
  choose!(**opts, &block)
148
+ rescue NoExperimentsError => e
149
+ raise e
147
150
  rescue => e
148
151
  TrailGuide.logger.error e
149
152
  experiment.control
@@ -158,13 +161,13 @@ module TrailGuide
158
161
  unless context.respond_to?(varmeth, true)
159
162
  if context_type == :controller
160
163
  raise NoVariantMethodError,
161
- "Undefined local method `#{varmeth}`. You must define a controller method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
164
+ "Undefined local method `#{varmeth}`. You must define a controller method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
162
165
  elsif context_type == :template
163
166
  raise NoVariantMethodError,
164
- "Undefined local method `#{varmeth}`. You must define a helper method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define ApplicationHelper##{varmeth}(metadata={})"
167
+ "Undefined local method `#{varmeth}`. You must define a helper method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define ApplicationHelper##{varmeth}(metadata={})"
165
168
  else
166
169
  raise NoVariantMethodError,
167
- "Undefined local method `#{varmeth}`. You must define a method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
170
+ "Undefined local method `#{varmeth}`. You must define a method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
168
171
  end
169
172
  end
170
173
 
@@ -181,6 +184,8 @@ module TrailGuide
181
184
 
182
185
  def run(methods: nil, **opts)
183
186
  run!(methods: methods, **opts)
187
+ rescue NoExperimentsError => e
188
+ raise e
184
189
  rescue => e
185
190
  TrailGuide.logger.error e
186
191
  false
@@ -202,19 +207,41 @@ module TrailGuide
202
207
 
203
208
  def render(prefix: nil, templates: nil, locals: {}, **opts)
204
209
  render!(prefix: prefix, templates: templates, locals: locals, **opts)
210
+ rescue NoExperimentsError => e
211
+ raise e
205
212
  rescue => e
206
213
  TrailGuide.logger.error e
207
214
  false
208
215
  end
209
216
 
210
217
  def convert!(checkpoint=nil, **opts, &block)
211
- checkpoints = experiments.map { |experiment| experiment.convert!(checkpoint, **opts) }
218
+ raise NoExperimentsError, key if experiments.empty?
219
+ checkpoints = experiments.map do |experiment|
220
+ ckpt = checkpoint || experiment.goals.find { |g| g == key }
221
+ if experiment.combined?
222
+ experiment.combined_experiments.map do |combo|
223
+ combo.convert!(ckpt, **opts)
224
+ end
225
+ else
226
+ experiment.convert!(ckpt, **opts)
227
+ end
228
+ end.flatten
229
+
212
230
  return false unless checkpoints.any?
231
+
213
232
  if block_given?
214
233
  yield checkpoints, opts[:metadata]
215
234
  else
216
235
  checkpoints
217
236
  end
237
+ rescue NoExperimentsError => e
238
+ unless TrailGuide.configuration.ignore_orphaned_groups?
239
+ trace = e.backtrace.find { |t| !t.match?(Regexp.new(__FILE__)) }
240
+ .to_s.split(Rails.root.to_s).last
241
+ .split(':').first(2).join(':')
242
+ TrailGuide.catalog.orphaned(key, trace)
243
+ end
244
+ false
218
245
  end
219
246
 
220
247
  def convert(checkpoint=nil, **opts, &block)
@@ -225,7 +252,7 @@ module TrailGuide
225
252
  end
226
253
 
227
254
  def experiments
228
- @experiments ||= TrailGuide.catalog.select(metric).map do |experiment|
255
+ @experiments ||= TrailGuide.catalog.select(key).map do |experiment|
229
256
  experiment.new(participant)
230
257
  end
231
258
  end
@@ -0,0 +1,24 @@
1
+ module TrailGuide
2
+ module Metrics
3
+ # represents a checkpoint in a funnel
4
+ # TODO this is mostly a placeholder for now
5
+ class Checkpoint < Goal
6
+ attr_reader :experiment, :name
7
+
8
+ def dup(experiment)
9
+ self.class.new(experiment, name)
10
+ end
11
+
12
+ def initialize(experiment, name, checkpoints=[])
13
+ @experiment = experiment
14
+ @name = name.to_s.underscore.to_sym
15
+ end
16
+
17
+ def as_json(opts={})
18
+ {
19
+ name: name,
20
+ }
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,45 @@
1
+ module TrailGuide
2
+ module Metrics
3
+ class Config < Canfig::Config
4
+ attr_reader :metric
5
+
6
+ INHERIT_KEYS = [ :allow_multiple_conversions, :allow_conversion, :on_convert ]
7
+ CALLBACK_KEYS = [ :allow_conversion, :on_convert ]
8
+
9
+ def initialize(metric, *args, **opts, &block)
10
+ @metric = metric
11
+
12
+ opts = INHERIT_KEYS.map do |key|
13
+ val = experiment.configuration[key]
14
+ val = val.dup if val
15
+ [ key, val ]
16
+ end.to_h.merge(opts)
17
+
18
+ super(*args, **opts, &block)
19
+ end
20
+
21
+ def experiment
22
+ metric.experiment
23
+ end
24
+
25
+ def callbacks
26
+ to_h.slice(*CALLBACK_KEYS).map { |k,v| [k, [v].flatten.compact] }.to_h
27
+ end
28
+
29
+ def allow_multiple_conversions?
30
+ !!allow_multiple_conversions
31
+ end
32
+
33
+ # TODO do we allow a method here? do we call it on the experiment?
34
+ def allow_conversion(meth=nil, &block)
35
+ self[:allow_conversion] ||= []
36
+ self[:allow_conversion] << (meth || block)
37
+ end
38
+
39
+ def on_convert(meth=nil, &block)
40
+ self[:on_convert] ||= []
41
+ self[:on_convert] << (meth || block)
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,24 @@
1
+ module TrailGuide
2
+ module Metrics
3
+ # represents a funnel of goal checkpoints
4
+ # TODO this is mostly a placeholder for now
5
+ class Funnel < Goal
6
+ attr_reader :experiment, :name
7
+
8
+ def dup(experiment)
9
+ self.class.new(experiment, name)
10
+ end
11
+
12
+ def initialize(experiment, name, checkpoints=[])
13
+ @experiment = experiment
14
+ @name = name.to_s.underscore.to_sym
15
+ end
16
+
17
+ def as_json(opts={})
18
+ {
19
+ name: name,
20
+ }
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,89 @@
1
+ module TrailGuide
2
+ module Metrics
3
+ # represents a simple conversion goal
4
+ class Goal
5
+ attr_reader :experiment, :name
6
+
7
+ delegate :allow_multiple_conversions?, :callbacks, to: :configuration
8
+
9
+ def dup(experiment)
10
+ self.class.new(experiment, name, **configuration.to_h.map { |k,v| [k, v.try(:dup)] }.to_h)
11
+ end
12
+
13
+ def configuration
14
+ @configuration ||= Metrics::Config.new(self)
15
+ end
16
+
17
+ def configure(*args, &block)
18
+ configuration.configure(*args, &block)
19
+ end
20
+
21
+ def initialize(experiment, name, **config, &block)
22
+ @experiment = experiment
23
+ @name = name.to_s.underscore.to_sym
24
+ configure(**config, &block)
25
+ end
26
+
27
+ def ==(other)
28
+ if other.is_a?(self.class)
29
+ return name == other.name
30
+ elsif other.is_a?(String) || other.is_a?(Symbol)
31
+ other = other.to_s.underscore
32
+ return name == other.to_sym || to_s == other
33
+ elsif other.is_a?(Array)
34
+ return to_s == other.flatten.map { |o| o.to_s.underscore }.join('/')
35
+ elsif other.is_a?(Hash)
36
+ # TODO "flatten" it out and compare it to_s
37
+ return false
38
+ end
39
+ end
40
+
41
+ def ===(other)
42
+ return false unless other.is_a?(self.class)
43
+ return name == other.name && experiment == other.experiment
44
+ end
45
+
46
+ def allow_conversion?(trial, variant, metadata=nil)
47
+ return true if callbacks[:allow_conversion].empty?
48
+ run_callbacks(:allow_conversion, trial, variant, trial.participant, metadata)
49
+ end
50
+
51
+ def run_callbacks(hook, trial, *args)
52
+ return unless callbacks[hook]
53
+ if [:allow_conversion].include?(hook)
54
+ callbacks[hook].reduce(args.slice!(0,1)[0]) do |result, callback|
55
+ if callback.respond_to?(:call)
56
+ callback.call(trial, result, self, *args)
57
+ else
58
+ trial.send(callback, trial, result, self, *args)
59
+ end
60
+ end
61
+ else
62
+ args.unshift(self)
63
+ args.unshift(trial)
64
+ callbacks[hook].each do |callback|
65
+ if callback.respond_to?(:call)
66
+ callback.call(*args)
67
+ else
68
+ trial.send(callback, *args)
69
+ end
70
+ end
71
+ end
72
+ end
73
+
74
+ def as_json(opts={})
75
+ {
76
+ name: name,
77
+ }
78
+ end
79
+
80
+ def to_s
81
+ name.to_s
82
+ end
83
+
84
+ def storage_key
85
+ "#{experiment.experiment_name}:#{name}"
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,9 @@
1
+ require 'trail_guide/metrics/config'
2
+ require 'trail_guide/metrics/goal'
3
+ require 'trail_guide/metrics/funnel'
4
+ require 'trail_guide/metrics/checkpoint'
5
+
6
+ module TrailGuide
7
+ module Metrics
8
+ end
9
+ end
@@ -6,7 +6,7 @@ module TrailGuide
6
6
  def initialize(context, adapter: nil)
7
7
  @context = context
8
8
  @adapter = adapter.new(context) if adapter.present?
9
- cleanup_inactive_experiments! if TrailGuide.configuration.cleanup_participant_experiments
9
+ cleanup_inactive_experiments! if TrailGuide.configuration.cleanup_participant_experiments == true
10
10
  end
11
11
 
12
12
  def adapter
@@ -36,14 +36,15 @@ module TrailGuide
36
36
  end
37
37
 
38
38
  def variant(experiment)
39
- return nil unless experiment.started?
39
+ return nil unless experiment.calibrating? || experiment.started?
40
40
  return nil unless adapter.key?(experiment.storage_key)
41
41
  varname = adapter[experiment.storage_key]
42
42
  variant = experiment.variants.find { |var| var == varname }
43
43
  return nil unless variant && adapter.key?(variant.storage_key)
44
44
 
45
45
  chosen_at = Time.at(adapter[variant.storage_key].to_i)
46
- return variant if chosen_at >= experiment.started_at
46
+ started_at = experiment.started_at
47
+ return variant if (variant.control? && experiment.calibrating?) || (started_at && chosen_at >= started_at)
47
48
  end
48
49
 
49
50
  def participating?(experiment, include_control=true)
@@ -54,27 +55,29 @@ module TrailGuide
54
55
  end
55
56
 
56
57
  def converted?(experiment, checkpoint=nil)
57
- return false unless experiment.started?
58
+ variant = variant(experiment)
59
+
60
+ return false unless experiment.started? || (experiment.calibrating? && variant.try(:control?))
61
+
58
62
  if experiment.goals.empty?
59
63
  raise InvalidGoalError, "You provided the checkpoint `#{checkpoint}` but the experiment `#{experiment.experiment_name}` does not have any goals defined." unless checkpoint.nil?
60
64
  storage_key = "#{experiment.storage_key}:converted"
61
65
  return false unless adapter.key?(storage_key)
62
66
 
63
67
  converted_at = Time.at(adapter[storage_key].to_i)
64
- converted_at >= experiment.started_at
68
+ (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
65
69
  elsif !checkpoint.nil?
66
- raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." unless experiment.goals.any? { |goal| goal == checkpoint.to_s.underscore.to_sym }
67
- storage_key = "#{experiment.storage_key}:#{checkpoint.to_s.underscore}"
68
- return false unless adapter.key?(storage_key)
70
+ goal = experiment.goals.find { |g| g == checkpoint }
71
+ raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." if goal.nil?
72
+ return false unless adapter.key?(goal.storage_key)
69
73
 
70
- converted_at = Time.at(adapter[storage_key].to_i)
71
- converted_at >= experiment.started_at
74
+ converted_at = Time.at(adapter[goal.storage_key].to_i)
75
+ (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
72
76
  else
73
77
  experiment.goals.each do |goal|
74
- storage_key = "#{experiment.storage_key}:#{goal.to_s}"
75
- next unless adapter.key?(storage_key)
76
- converted_at = Time.at(adapter[storage_key].to_i)
77
- return true if converted_at >= experiment.started_at
78
+ next unless adapter.key?(goal.storage_key)
79
+ converted_at = Time.at(adapter[goal.storage_key].to_i)
80
+ return true if (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
78
81
  end
79
82
  return false
80
83
  end
@@ -86,16 +89,18 @@ module TrailGuide
86
89
  end
87
90
 
88
91
  def converted!(variant, checkpoint=nil, reset: false)
89
- checkpoint ||= :converted
90
- storage_key = "#{variant.experiment.storage_key}:#{checkpoint.to_s.underscore}"
92
+ if checkpoint.nil?
93
+ storage_key = "#{variant.experiment.storage_key}:converted"
94
+ else
95
+ storage_key = variant.experiment.goals.find { |g| g == checkpoint }.storage_key
96
+ end
91
97
 
92
98
  if reset
93
99
  adapter.delete(variant.experiment.storage_key)
94
100
  adapter.delete(variant.storage_key)
95
101
  adapter.delete(storage_key)
96
102
  variant.experiment.goals.each do |goal|
97
- goal_key = "#{variant.experiment.storage_key}:#{goal.to_s}"
98
- adapter.delete(goal_key)
103
+ adapter.delete(goal.storage_key)
99
104
  end
100
105
  else
101
106
  adapter[storage_key] = Time.now.to_i
@@ -107,17 +112,45 @@ module TrailGuide
107
112
  return true if chosen.nil?
108
113
  adapter.delete(experiment.storage_key)
109
114
  adapter.delete(chosen.storage_key)
115
+ adapter.delete("#{experiment.storage_key}:converted")
110
116
  experiment.goals.each do |goal|
111
- adapter.delete("#{experiment.storage_key}:#{goal.to_s}")
117
+ adapter.delete(goal.storage_key)
112
118
  end
113
119
  return true
114
120
  end
115
121
 
116
122
  def active_experiments(include_control=true)
117
123
  return false if adapter.keys.empty?
124
+
125
+ inactive = []
126
+ active = adapter.keys.map { |key| key.to_s.split(":").first.to_sym }.uniq.map do |key|
127
+ experiment = TrailGuide.catalog.find(key)
128
+ next unless experiment
129
+
130
+ if !experiment.started? && !experiment.calibrating?
131
+ inactive << key
132
+ next
133
+ else
134
+ next unless !experiment.combined? && experiment.running? && participating?(experiment, include_control)
135
+ [ experiment.experiment_name, adapter[experiment.storage_key] ]
136
+ end
137
+ end.compact.to_h
138
+
139
+ if TrailGuide.configuration.cleanup_participant_experiments == :inline && !inactive.empty?
140
+ adapter.keys.select do |key|
141
+ inactive.include?(key.to_s.split(":").first.to_sym)
142
+ end.each { |key| adapter.delete(key) }
143
+ end
144
+
145
+ return active
146
+ end
147
+
148
+ def calibrating_experiments
149
+ return false if adapter.keys.empty?
150
+
118
151
  adapter.keys.map { |key| key.to_s.split(":").first.to_sym }.uniq.map do |key|
119
152
  experiment = TrailGuide.catalog.find(key)
120
- next unless experiment && !experiment.combined? && experiment.running? && participating?(experiment, include_control)
153
+ next unless experiment && experiment.calibrating?
121
154
  [ experiment.experiment_name, adapter[experiment.storage_key] ]
122
155
  end.compact.to_h
123
156
  end
@@ -138,7 +171,7 @@ module TrailGuide
138
171
  adapter.keys.each do |key|
139
172
  experiment_name = key.to_s.split(":").first.to_sym
140
173
  experiment = TrailGuide.catalog.find(experiment_name)
141
- if !experiment || !experiment.started?
174
+ if !experiment || (!experiment.started? && !experiment.calibrating?)
142
175
  adapter.delete(key)
143
176
  end
144
177
  end
@@ -16,12 +16,19 @@ module TrailGuide
16
16
 
17
17
  def ==(other)
18
18
  if other.is_a?(self.class)
19
+ # TODO eventually remove the experiment requirement here once we start
20
+ # taking advantage of === below
19
21
  return name == other.name && experiment == other.experiment
20
22
  elsif other.is_a?(String) || other.is_a?(Symbol)
21
23
  return name == other.to_s.underscore.to_sym
22
24
  end
23
25
  end
24
26
 
27
+ def ===(other)
28
+ return false unless other.is_a?(self.class)
29
+ return name == other.name && experiment == other.experiment
30
+ end
31
+
25
32
  # TODO maybe track the control on the experiment itself, rather than as a
26
33
  # flag on the variants like this?
27
34
 
@@ -66,11 +73,12 @@ module TrailGuide
66
73
  raise InvalidGoalError, "You provided the checkpoint `#{checkpoint}` but the experiment `#{experiment.experiment_name}` does not have any goals defined." unless checkpoint.nil?
67
74
  (TrailGuide.redis.hget(storage_key, 'converted') || 0).to_i
68
75
  elsif !checkpoint.nil?
69
- raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." unless experiment.goals.any? { |goal| goal == checkpoint.to_s.underscore.to_sym }
70
- (TrailGuide.redis.hget(storage_key, checkpoint.to_s.underscore) || 0).to_i
76
+ goal = experiment.goals.find { |g| g == checkpoint }
77
+ raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." if goal.nil?
78
+ (TrailGuide.redis.hget(storage_key, goal.to_s) || 0).to_i
71
79
  else
72
- experiment.goals.sum do |checkpoint|
73
- (TrailGuide.redis.hget(storage_key, checkpoint.to_s.underscore) || 0).to_i
80
+ experiment.goals.sum do |goal|
81
+ (TrailGuide.redis.hget(storage_key, goal.to_s) || 0).to_i
74
82
  end
75
83
  end
76
84
  end
@@ -79,22 +87,36 @@ module TrailGuide
79
87
  participants - converted
80
88
  end
81
89
 
90
+ def measure(goal=nil, against=nil)
91
+ superset = against ? converted(against) : participants
92
+ converts = converted(goal)
93
+ return 0 if superset.zero? || converts.zero?
94
+ converts.to_f / superset.to_f
95
+ end
96
+
82
97
  def increment_participation!
83
98
  TrailGuide.redis.hincrby(storage_key, 'participants', 1)
84
99
  end
85
100
 
86
101
  def increment_conversion!(checkpoint=nil)
87
- checkpoint ||= :converted
88
- TrailGuide.redis.hincrby(storage_key, checkpoint.to_s.underscore, 1)
102
+ if checkpoint.nil?
103
+ checkpoint = 'converted'
104
+ else
105
+ checkpoint = experiment.goals.find { |g| g == checkpoint }.to_s
106
+ end
107
+ TrailGuide.redis.hincrby(storage_key, checkpoint, 1)
89
108
  end
90
109
 
110
+ # export the variant state (not config) as json
91
111
  def as_json(opts={})
92
- {
93
- name: name,
94
- control: control?,
95
- weight: weight,
96
- metadata: metadata.as_json,
97
- }
112
+ if experiment.goals.empty?
113
+ conversions = converted
114
+ else
115
+ conversions = experiment.goals.map { |g| [g.name, converted(g)] }.to_h
116
+ end
117
+
118
+ { name => { participants: participants,
119
+ converted: conversions } }
98
120
  end
99
121
 
100
122
  def to_s
@@ -1,8 +1,8 @@
1
1
  module TrailGuide
2
2
  module Version
3
3
  MAJOR = 0
4
- MINOR = 1
5
- PATCH = 31
4
+ MINOR = 2
5
+ PATCH = 0
6
6
  VERSION = "#{MAJOR}.#{MINOR}.#{PATCH}"
7
7
 
8
8
  class << self
data/lib/trailguide.rb CHANGED
@@ -5,6 +5,8 @@ require "trail_guide/config"
5
5
  require "trail_guide/errors"
6
6
  require "trail_guide/adapters"
7
7
  require "trail_guide/algorithms"
8
+ require "trail_guide/metrics"
9
+ require "trail_guide/calculators"
8
10
  require "trail_guide/participant"
9
11
  require "trail_guide/variant"
10
12
  require "trail_guide/experiment"
@@ -18,6 +20,8 @@ module TrailGuide
18
20
  include Canfig::Module
19
21
  @@configuration = TrailGuide::Config.new
20
22
 
23
+ SCHEDULE_DATE_FORMAT = "%m/%d/%Y %I:%M %p %Z"
24
+
21
25
  class << self
22
26
  delegate :logger, :redis, to: :configuration
23
27
  end