moses-vanity 1.7.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. data/.autotest +22 -0
  2. data/.gitignore +7 -0
  3. data/.rvmrc +3 -0
  4. data/.travis.yml +13 -0
  5. data/CHANGELOG +374 -0
  6. data/Gemfile +28 -0
  7. data/MIT-LICENSE +21 -0
  8. data/README.rdoc +108 -0
  9. data/Rakefile +189 -0
  10. data/bin/vanity +16 -0
  11. data/doc/_config.yml +2 -0
  12. data/doc/_layouts/_header.html +34 -0
  13. data/doc/_layouts/page.html +47 -0
  14. data/doc/_metrics.textile +12 -0
  15. data/doc/ab_testing.textile +210 -0
  16. data/doc/configuring.textile +45 -0
  17. data/doc/contributing.textile +93 -0
  18. data/doc/credits.textile +23 -0
  19. data/doc/css/page.css +83 -0
  20. data/doc/css/print.css +43 -0
  21. data/doc/css/syntax.css +7 -0
  22. data/doc/email.textile +129 -0
  23. data/doc/experimental.textile +31 -0
  24. data/doc/faq.textile +8 -0
  25. data/doc/identity.textile +43 -0
  26. data/doc/images/ab_in_dashboard.png +0 -0
  27. data/doc/images/clear_winner.png +0 -0
  28. data/doc/images/price_options.png +0 -0
  29. data/doc/images/sidebar_test.png +0 -0
  30. data/doc/images/signup_metric.png +0 -0
  31. data/doc/images/vanity.png +0 -0
  32. data/doc/index.textile +91 -0
  33. data/doc/metrics.textile +231 -0
  34. data/doc/rails.textile +89 -0
  35. data/doc/site.js +27 -0
  36. data/generators/templates/vanity_migration.rb +53 -0
  37. data/generators/vanity_generator.rb +8 -0
  38. data/lib/generators/templates/vanity_migration.rb +53 -0
  39. data/lib/generators/vanity_generator.rb +15 -0
  40. data/lib/vanity.rb +36 -0
  41. data/lib/vanity/adapters/abstract_adapter.rb +140 -0
  42. data/lib/vanity/adapters/active_record_adapter.rb +248 -0
  43. data/lib/vanity/adapters/mock_adapter.rb +157 -0
  44. data/lib/vanity/adapters/mongodb_adapter.rb +178 -0
  45. data/lib/vanity/adapters/redis_adapter.rb +160 -0
  46. data/lib/vanity/backport.rb +26 -0
  47. data/lib/vanity/commands/list.rb +21 -0
  48. data/lib/vanity/commands/report.rb +64 -0
  49. data/lib/vanity/commands/upgrade.rb +34 -0
  50. data/lib/vanity/experiment/ab_test.rb +507 -0
  51. data/lib/vanity/experiment/base.rb +214 -0
  52. data/lib/vanity/frameworks.rb +16 -0
  53. data/lib/vanity/frameworks/rails.rb +318 -0
  54. data/lib/vanity/helpers.rb +66 -0
  55. data/lib/vanity/images/x.gif +0 -0
  56. data/lib/vanity/metric/active_record.rb +85 -0
  57. data/lib/vanity/metric/base.rb +244 -0
  58. data/lib/vanity/metric/google_analytics.rb +83 -0
  59. data/lib/vanity/metric/remote.rb +53 -0
  60. data/lib/vanity/playground.rb +396 -0
  61. data/lib/vanity/templates/_ab_test.erb +28 -0
  62. data/lib/vanity/templates/_experiment.erb +5 -0
  63. data/lib/vanity/templates/_experiments.erb +7 -0
  64. data/lib/vanity/templates/_metric.erb +14 -0
  65. data/lib/vanity/templates/_metrics.erb +13 -0
  66. data/lib/vanity/templates/_report.erb +27 -0
  67. data/lib/vanity/templates/_vanity.js.erb +20 -0
  68. data/lib/vanity/templates/flot.min.js +1 -0
  69. data/lib/vanity/templates/jquery.min.js +19 -0
  70. data/lib/vanity/templates/vanity.css +26 -0
  71. data/lib/vanity/templates/vanity.js +82 -0
  72. data/lib/vanity/version.rb +11 -0
  73. data/test/adapters/redis_adapter_test.rb +17 -0
  74. data/test/experiment/ab_test.rb +771 -0
  75. data/test/experiment/base_test.rb +150 -0
  76. data/test/experiments/age_and_zipcode.rb +19 -0
  77. data/test/experiments/metrics/cheers.rb +3 -0
  78. data/test/experiments/metrics/signups.rb +2 -0
  79. data/test/experiments/metrics/yawns.rb +3 -0
  80. data/test/experiments/null_abc.rb +5 -0
  81. data/test/metric/active_record_test.rb +277 -0
  82. data/test/metric/base_test.rb +293 -0
  83. data/test/metric/google_analytics_test.rb +104 -0
  84. data/test/metric/remote_test.rb +109 -0
  85. data/test/myapp/app/controllers/application_controller.rb +2 -0
  86. data/test/myapp/app/controllers/main_controller.rb +7 -0
  87. data/test/myapp/config/boot.rb +110 -0
  88. data/test/myapp/config/environment.rb +10 -0
  89. data/test/myapp/config/environments/production.rb +0 -0
  90. data/test/myapp/config/routes.rb +3 -0
  91. data/test/passenger_test.rb +43 -0
  92. data/test/playground_test.rb +26 -0
  93. data/test/rails_dashboard_test.rb +37 -0
  94. data/test/rails_helper_test.rb +36 -0
  95. data/test/rails_test.rb +389 -0
  96. data/test/test_helper.rb +145 -0
  97. data/vanity.gemspec +26 -0
  98. metadata +202 -0
@@ -0,0 +1,66 @@
1
+ module Vanity
2
+ # Helper methods available on Object.
3
+ #
4
+ # @example From ERB template
5
+ # <%= ab_test(:greeting) %> <%= current_user.name %>
6
+ # @example From Rails controller
7
+ # class AccountController < ApplicationController
8
+ # def create
9
+ # track! :signup
10
+ # Acccount.create! params[:account]
11
+ # end
12
+ # end
13
+ # @example From ActiveRecord
14
+ # class Posts < ActiveRecord::Base
15
+ # after_create do |post|
16
+ # track! :images if post.type == :image
17
+ # end
18
+ # end
19
+ module Helpers
20
+
21
+ # This method returns one of the alternative values in the named A/B test.
22
+ #
23
+ # @example A/B two alternatives for a page
24
+ # def index
25
+ # if ab_test(:new_page) # true/false test
26
+ # render action: "new_page"
27
+ # else
28
+ # render action: "index"
29
+ # end
30
+ # end
31
+ # @example Similar, alternative value is page name
32
+ # def index
33
+ # render action: ab_test(:new_page)
34
+ # end
35
+ # @since 1.2.0
36
+ def ab_test(name, &block)
37
+ if Vanity.playground.using_js?
38
+ @_vanity_experiments ||= {}
39
+ @_vanity_experiments[name] ||= Vanity.playground.experiment(name).choose
40
+ value = @_vanity_experiments[name].value
41
+ else
42
+ value = Vanity.playground.experiment(name).choose.value
43
+ end
44
+
45
+ if block
46
+ content = capture(value, &block)
47
+ block_called_from_erb?(block) ? concat(content) : content
48
+ else
49
+ value
50
+ end
51
+ end
52
+
53
+ # Tracks an action associated with a metric.
54
+ #
55
+ # @example
56
+ # track! :invitation
57
+ # @since 1.2.0
58
+ def track!(name, count = 1)
59
+ Vanity.playground.track! name, count
60
+ end
61
+ end
62
+ end
63
+
64
+ Object.class_eval do
65
+ include Vanity::Helpers
66
+ end
Binary file
@@ -0,0 +1,85 @@
1
+ module Vanity
2
+ class Metric
3
+
4
+ AGGREGATES = [:average, :minimum, :maximum, :sum]
5
+
6
+ # Use an ActiveRecord model to get metric data from database table. Also
7
+ # forwards +after_create+ callbacks to hooks (updating experiments).
8
+ #
9
+ # Supported options:
10
+ # :conditions -- Only select records that match this condition
11
+ # :average -- Metric value is average of this column
12
+ # :minimum -- Metric value is minimum of this column
13
+ # :maximum -- Metric value is maximum of this column
14
+ # :sum -- Metric value is sum of this column
15
+ # :timestamp -- Use this column to filter/group records (defaults to
16
+ # +created_at+)
17
+ #
18
+ # @example Track sign ups using User model
19
+ # metric "Signups" do
20
+ # model Account
21
+ # end
22
+ # @example Track satisfaction using Survey model
23
+ # metric "Satisfaction" do
24
+ # model Survey, :average=>:rating
25
+ # end
26
+ # @example Track only high ratings
27
+ # metric "High ratings" do
28
+ # model Rating, :conditions=>["stars >= 4"]
29
+ # end
30
+ # @example Track only high ratings (using scope)
31
+ # metric "High ratings" do
32
+ # model Rating.high
33
+ # end
34
+ #
35
+ # @since 1.2.0
36
+ # @see Vanity::Metric::ActiveRecord
37
+ def model(class_or_scope, options = nil)
38
+ options = options || {}
39
+ conditions = options.delete(:conditions)
40
+ @ar_scoped = conditions ? class_or_scope.scoped(:conditions=>conditions) : class_or_scope
41
+ @ar_aggregate = AGGREGATES.find { |key| options.has_key?(key) }
42
+ @ar_column = options.delete(@ar_aggregate)
43
+ fail "Cannot use multiple aggregates in a single metric" if AGGREGATES.find { |key| options.has_key?(key) }
44
+ @ar_timestamp = options.delete(:timestamp) || :created_at
45
+ @ar_timestamp, @ar_timestamp_table = @ar_timestamp.to_s.split('.').reverse
46
+ @ar_timestamp_table ||= @ar_scoped.table_name
47
+ fail "Unrecognized options: #{options.keys * ", "}" unless options.empty?
48
+ @ar_scoped.after_create self
49
+ extend ActiveRecord
50
+ end
51
+
52
+ # Calling model method on a metric extends it with these modules, redefining
53
+ # the values and track! methods.
54
+ #
55
+ # @since 1.3.0
56
+ module ActiveRecord
57
+
58
+ # This values method queries the database.
59
+ def values(sdate, edate)
60
+ query = { :conditions=> { @ar_timestamp_table => { @ar_timestamp => (sdate.to_time...(edate + 1).to_time) } },
61
+ :group=>"date(#{@ar_scoped.quoted_table_name}.#{@ar_scoped.connection.quote_column_name @ar_timestamp})" }
62
+ grouped = @ar_column ? @ar_scoped.send(@ar_aggregate, @ar_column, query) : @ar_scoped.count(query)
63
+ (sdate..edate).inject([]) { |ordered, date| ordered << (grouped[date.to_s] || 0) }
64
+ end
65
+
66
+ # This track! method stores nothing, but calls the hooks.
67
+ def track!(args = nil)
68
+ return unless @playground.collecting?
69
+ call_hooks *track_args(args)
70
+ end
71
+
72
+ def last_update_at
73
+ record = @ar_scoped.find(:first, :order=>"#@ar_timestamp DESC", :limit=>1, :select=>@ar_timestamp)
74
+ record && record.send(@ar_timestamp)
75
+ end
76
+
77
+ # AR model after_create callback notifies all the hooks.
78
+ def after_create(record)
79
+ return unless @playground.collecting?
80
+ count = @ar_column ? (record.send(@ar_column) || 0) : 1
81
+ call_hooks record.send(@ar_timestamp), nil, [count] if count > 0 && @ar_scoped.exists?(record)
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,244 @@
1
+ module Vanity
2
+
3
+ # A metric is an object that implements two methods: +name+ and +values+. It
4
+ # can also respond to addition methods (+track!+, +bounds+, etc), these are
5
+ # optional.
6
+ #
7
+ # This class implements a basic metric that tracks data and stores it in the
8
+ # database. You can use this as the basis for your metric, or as reference
9
+ # for the methods your metric must and can implement.
10
+ #
11
+ # @since 1.1.0
12
+ class Metric
13
+
14
+ # These methods are available when defining a metric in a file loaded
15
+ # from the +experiments/metrics+ directory.
16
+ #
17
+ # For example:
18
+ # $ cat experiments/metrics/yawn_sec
19
+ # metric "Yawns/sec" do
20
+ # description "Most boring metric ever"
21
+ # end
22
+ module Definition
23
+
24
+ attr_reader :playground
25
+
26
+ # Defines a new metric, using the class Vanity::Metric.
27
+ def metric(name, &block)
28
+ fail "Metric #{@metric_id} already defined in playground" if playground.metrics[@metric_id]
29
+ metric = Metric.new(playground, name.to_s, @metric_id)
30
+ metric.instance_eval &block
31
+ playground.metrics[@metric_id] = metric
32
+ end
33
+
34
+ def new_binding(playground, id)
35
+ @playground, @metric_id = playground, id
36
+ binding
37
+ end
38
+
39
+ end
40
+
41
+ # Startup metrics for pirates. AARRR stands for:
42
+ # * Acquisition
43
+ # * Activation
44
+ # * Retention
45
+ # * Referral
46
+ # * Revenue
47
+ # Read more: http://500hats.typepad.com/500blogs/2007/09/startup-metrics.html
48
+
49
+ class << self
50
+
51
+ # Helper method to return description for a metric.
52
+ #
53
+ # A metric object may have a +description+ method that returns a detailed
54
+ # description. It may also have no description, or no +description+
55
+ # method, in which case return +nil+.
56
+ #
57
+ # @example
58
+ # puts Vanity::Metric.description(metric)
59
+ def description(metric)
60
+ metric.description if metric.respond_to?(:description)
61
+ end
62
+
63
+ # Helper method to return bounds for a metric.
64
+ #
65
+ # A metric object may have a +bounds+ method that returns lower and upper
66
+ # bounds. It may also have no bounds, or no +bounds+ # method, in which
67
+ # case we return +[nil, nil]+.
68
+ #
69
+ # @example
70
+ # upper = Vanity::Metric.bounds(metric).last
71
+ def bounds(metric)
72
+ metric.respond_to?(:bounds) && metric.bounds || [nil, nil]
73
+ end
74
+
75
+ # Returns data set for a given date range. The data set is an array of
76
+ # date, value pairs.
77
+ #
78
+ # First argument is the metric. Second argument is the start date, or
79
+ # number of days to go back in history, defaults to 90 days. Third
80
+ # argument is end date, defaults to today.
81
+ #
82
+ # @example These are all equivalent:
83
+ # Vanity::Metric.data(my_metric)
84
+ # Vanity::Metric.data(my_metric, 90)
85
+ # Vanity::Metric.data(my_metric, Date.today - 89)
86
+ # Vanity::Metric.data(my_metric, Date.today - 89, Date.today)
87
+ def data(metric, *args)
88
+ first = args.shift || 90
89
+ to = args.shift || Date.today
90
+ from = first.respond_to?(:to_date) ? first.to_date : to - (first - 1)
91
+ (from..to).zip(metric.values(from, to))
92
+ end
93
+
94
+ # Playground uses this to load metric definitions.
95
+ def load(playground, stack, file)
96
+ fail "Circular dependency detected: #{stack.join('=>')}=>#{file}" if stack.include?(file)
97
+ source = File.read(file)
98
+ stack.push file
99
+ id = File.basename(file, ".rb").downcase.gsub(/\W/, "_").to_sym
100
+ context = Object.new
101
+ context.instance_eval do
102
+ extend Definition
103
+ metric = eval(source, context.new_binding(playground, id), file)
104
+ fail NameError.new("Expected #{file} to define metric #{id}", id) unless playground.metrics[id]
105
+ metric
106
+ end
107
+ rescue
108
+ error = NameError.exception($!.message, id)
109
+ error.set_backtrace $!.backtrace
110
+ raise error
111
+ ensure
112
+ stack.pop
113
+ end
114
+
115
+ end
116
+
117
+
118
+ # Takes playground (need this to access Redis), friendly name and optional
119
+ # id (can infer from name).
120
+ def initialize(playground, name, id = nil)
121
+ @playground, @name = playground, name.to_s
122
+ @id = (id || name.to_s.downcase.gsub(/\W+/, '_')).to_sym
123
+ @hooks = []
124
+ end
125
+
126
+
127
+ # -- Tracking --
128
+
129
+ # Called to track an action associated with this metric. Most common is not
130
+ # passing an argument, and it tracks a count of 1. You can pass a different
131
+ # value as the argument, or array of value (for multi-series metrics), or
132
+ # hash with the optional keys timestamp, identity and values.
133
+ #
134
+ # Example:
135
+ # hits.track!
136
+ # foo_and_bar.track! [5,11]
137
+ def track!(args = nil)
138
+ return unless @playground.collecting?
139
+ timestamp, identity, values = track_args(args)
140
+ connection.metric_track @id, timestamp, identity, values
141
+ @playground.logger.info "vanity: #{@id} with value #{values.join(", ")}"
142
+ call_hooks timestamp, identity, values
143
+ end
144
+
145
+ # Parses arguments to track! method and return array with timestamp,
146
+ # identity and array of values.
147
+ def track_args(args)
148
+ case args
149
+ when Hash
150
+ timestamp, identity, values = args.values_at(:timestamp, :identity, :values)
151
+ when Array
152
+ values = args
153
+ when Numeric
154
+ values = [args]
155
+ end
156
+ identity = Vanity.context.vanity_identity rescue nil
157
+ [timestamp || Time.now, identity, values || [1]]
158
+ end
159
+ protected :track_args
160
+
161
+ # Metric definitions use this to introduce tracking hook. The hook is
162
+ # called with metric identifier, timestamp, count and possibly additional
163
+ # arguments.
164
+ #
165
+ # For example:
166
+ # hook do |metric_id, timestamp, count|
167
+ # syslog.info metric_id
168
+ # end
169
+ def hook(&block)
170
+ @hooks << block
171
+ end
172
+
173
+ # This method returns the acceptable bounds of a metric as an array with
174
+ # two values: low and high. Use nil for unbounded.
175
+ #
176
+ # Alerts are created when metric values exceed their bounds. For example,
177
+ # a metric of user registration can use historical data to calculate
178
+ # expected range of new registration for the next day. If actual metric
179
+ # falls below the expected range, it could indicate registration process is
180
+ # broken. Going above higher bound could trigger opening a Champagne
181
+ # bottle.
182
+ #
183
+ # The default implementation returns +nil+.
184
+ def bounds
185
+ end
186
+
187
+
188
+ # -- Reporting --
189
+
190
+ # Human readable metric name. All metrics must implement this method.
191
+ attr_reader :name
192
+ alias :to_s :name
193
+
194
+ # Human readable description. Use two newlines to break paragraphs.
195
+ attr_accessor :description
196
+
197
+ # Sets or returns description. For example
198
+ # metric "Yawns/sec" do
199
+ # description "Most boring metric ever"
200
+ # end
201
+ #
202
+ # puts "Just defined: " + metric(:boring).description
203
+ def description(text = nil)
204
+ @description = text if text
205
+ @description
206
+ end
207
+
208
+ # Given two arguments, a start date and an end date (inclusive), returns an
209
+ # array of measurements. All metrics must implement this method.
210
+ def values(from, to)
211
+ values = connection.metric_values(@id, from, to)
212
+ values.map { |row| row.first.to_i }
213
+ end
214
+
215
+ # Returns date/time of the last update to this metric.
216
+ #
217
+ # @since 1.4.0
218
+ def last_update_at
219
+ connection.get_metric_last_update_at(@id)
220
+ end
221
+
222
+
223
+ # -- Storage --
224
+
225
+ def destroy!
226
+ connection.destroy_metric @id
227
+ end
228
+
229
+ def connection
230
+ @playground.connection
231
+ end
232
+
233
+ def key(*args)
234
+ "metrics:#{@id}:#{args.join(':')}"
235
+ end
236
+
237
+ def call_hooks(timestamp, identity, values)
238
+ @hooks.each do |hook|
239
+ hook.call @id, timestamp, values.first || 1
240
+ end
241
+ end
242
+
243
+ end
244
+ end
@@ -0,0 +1,83 @@
1
+ module Vanity
2
+ class Metric
3
+
4
+ # Use Google Analytics metric. Note: you must +require "garb"+ before
5
+ # vanity.
6
+ #
7
+ # @example Page views
8
+ # metric "Page views" do
9
+ # google_analytics "UA-1828623-6"
10
+ # end
11
+ # @example Visits
12
+ # metric "Visits" do
13
+ # google_analytics "UA-1828623-6", :visits
14
+ # end
15
+ #
16
+ # @since 1.3.0
17
+ # @see Vanity::Metric::GoogleAnalytics
18
+ def google_analytics(web_property_id, *args)
19
+ require "garb"
20
+ options = Hash === args.last ? args.pop : {}
21
+ metric = args.shift || :pageviews
22
+ @ga_resource = Vanity::Metric::GoogleAnalytics::Resource.new(web_property_id, metric)
23
+ @ga_mapper = options[:mapper] ||= lambda { |entry| entry.send(@ga_resource.metrics.elements.first).to_i }
24
+ extend GoogleAnalytics
25
+ rescue LoadError
26
+ fail LoadError, "Google Analytics metrics require Garb, please gem install garb first"
27
+ end
28
+
29
+ # Calling google_analytics method on a metric extends it with these modules,
30
+ # redefining the values and hook methods.
31
+ #
32
+ # @since 1.3.0
33
+ module GoogleAnalytics
34
+
35
+ # Returns values from GA using parameters specified by prior call to
36
+ # google_analytics.
37
+ def values(from, to)
38
+ data = @ga_resource.results(from, to).inject({}) do |hash,entry|
39
+ hash.merge(entry.date=>@ga_mapper.call(entry))
40
+ end
41
+ (from..to).map { |day| data[day.strftime('%Y%m%d')] || 0 }
42
+ end
43
+
44
+ # Hooks not supported for GA metrics.
45
+ def hook
46
+ fail "Cannot use hooks with Google Analytics methods"
47
+ end
48
+
49
+ # Garb report.
50
+ def report
51
+ @ga_resource
52
+ end
53
+
54
+ # Unkown (for now).
55
+ def last_update_at
56
+ end
57
+
58
+ def track!(args = nil)
59
+ end
60
+
61
+ class Resource
62
+ # GA profile used for this report. Populated after calling results.
63
+ attr_reader :profile
64
+
65
+ def initialize(web_property_id, metric)
66
+ self.class.send :include, Garb::Resource
67
+ @web_property_id = web_property_id
68
+ metrics metric
69
+ dimensions :date
70
+ sort :date
71
+ end
72
+
73
+ def results(start_date, end_date)
74
+ @profile = Garb::Profile.all.find { |p| p.web_property_id == @web_property_id }
75
+ @start_date = start_date
76
+ @end_date = end_date
77
+ Garb::ReportResponse.new(send_request_for_body).results
78
+ end
79
+ end
80
+
81
+ end
82
+ end
83
+ end