moses-vanity 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. data/.autotest +22 -0
  2. data/.gitignore +7 -0
  3. data/.rvmrc +3 -0
  4. data/.travis.yml +13 -0
  5. data/CHANGELOG +374 -0
  6. data/Gemfile +28 -0
  7. data/MIT-LICENSE +21 -0
  8. data/README.rdoc +108 -0
  9. data/Rakefile +189 -0
  10. data/bin/vanity +16 -0
  11. data/doc/_config.yml +2 -0
  12. data/doc/_layouts/_header.html +34 -0
  13. data/doc/_layouts/page.html +47 -0
  14. data/doc/_metrics.textile +12 -0
  15. data/doc/ab_testing.textile +210 -0
  16. data/doc/configuring.textile +45 -0
  17. data/doc/contributing.textile +93 -0
  18. data/doc/credits.textile +23 -0
  19. data/doc/css/page.css +83 -0
  20. data/doc/css/print.css +43 -0
  21. data/doc/css/syntax.css +7 -0
  22. data/doc/email.textile +129 -0
  23. data/doc/experimental.textile +31 -0
  24. data/doc/faq.textile +8 -0
  25. data/doc/identity.textile +43 -0
  26. data/doc/images/ab_in_dashboard.png +0 -0
  27. data/doc/images/clear_winner.png +0 -0
  28. data/doc/images/price_options.png +0 -0
  29. data/doc/images/sidebar_test.png +0 -0
  30. data/doc/images/signup_metric.png +0 -0
  31. data/doc/images/vanity.png +0 -0
  32. data/doc/index.textile +91 -0
  33. data/doc/metrics.textile +231 -0
  34. data/doc/rails.textile +89 -0
  35. data/doc/site.js +27 -0
  36. data/generators/templates/vanity_migration.rb +53 -0
  37. data/generators/vanity_generator.rb +8 -0
  38. data/lib/generators/templates/vanity_migration.rb +53 -0
  39. data/lib/generators/vanity_generator.rb +15 -0
  40. data/lib/vanity.rb +36 -0
  41. data/lib/vanity/adapters/abstract_adapter.rb +140 -0
  42. data/lib/vanity/adapters/active_record_adapter.rb +248 -0
  43. data/lib/vanity/adapters/mock_adapter.rb +157 -0
  44. data/lib/vanity/adapters/mongodb_adapter.rb +178 -0
  45. data/lib/vanity/adapters/redis_adapter.rb +160 -0
  46. data/lib/vanity/backport.rb +26 -0
  47. data/lib/vanity/commands/list.rb +21 -0
  48. data/lib/vanity/commands/report.rb +64 -0
  49. data/lib/vanity/commands/upgrade.rb +34 -0
  50. data/lib/vanity/experiment/ab_test.rb +507 -0
  51. data/lib/vanity/experiment/base.rb +214 -0
  52. data/lib/vanity/frameworks.rb +16 -0
  53. data/lib/vanity/frameworks/rails.rb +318 -0
  54. data/lib/vanity/helpers.rb +66 -0
  55. data/lib/vanity/images/x.gif +0 -0
  56. data/lib/vanity/metric/active_record.rb +85 -0
  57. data/lib/vanity/metric/base.rb +244 -0
  58. data/lib/vanity/metric/google_analytics.rb +83 -0
  59. data/lib/vanity/metric/remote.rb +53 -0
  60. data/lib/vanity/playground.rb +396 -0
  61. data/lib/vanity/templates/_ab_test.erb +28 -0
  62. data/lib/vanity/templates/_experiment.erb +5 -0
  63. data/lib/vanity/templates/_experiments.erb +7 -0
  64. data/lib/vanity/templates/_metric.erb +14 -0
  65. data/lib/vanity/templates/_metrics.erb +13 -0
  66. data/lib/vanity/templates/_report.erb +27 -0
  67. data/lib/vanity/templates/_vanity.js.erb +20 -0
  68. data/lib/vanity/templates/flot.min.js +1 -0
  69. data/lib/vanity/templates/jquery.min.js +19 -0
  70. data/lib/vanity/templates/vanity.css +26 -0
  71. data/lib/vanity/templates/vanity.js +82 -0
  72. data/lib/vanity/version.rb +11 -0
  73. data/test/adapters/redis_adapter_test.rb +17 -0
  74. data/test/experiment/ab_test.rb +771 -0
  75. data/test/experiment/base_test.rb +150 -0
  76. data/test/experiments/age_and_zipcode.rb +19 -0
  77. data/test/experiments/metrics/cheers.rb +3 -0
  78. data/test/experiments/metrics/signups.rb +2 -0
  79. data/test/experiments/metrics/yawns.rb +3 -0
  80. data/test/experiments/null_abc.rb +5 -0
  81. data/test/metric/active_record_test.rb +277 -0
  82. data/test/metric/base_test.rb +293 -0
  83. data/test/metric/google_analytics_test.rb +104 -0
  84. data/test/metric/remote_test.rb +109 -0
  85. data/test/myapp/app/controllers/application_controller.rb +2 -0
  86. data/test/myapp/app/controllers/main_controller.rb +7 -0
  87. data/test/myapp/config/boot.rb +110 -0
  88. data/test/myapp/config/environment.rb +10 -0
  89. data/test/myapp/config/environments/production.rb +0 -0
  90. data/test/myapp/config/routes.rb +3 -0
  91. data/test/passenger_test.rb +43 -0
  92. data/test/playground_test.rb +26 -0
  93. data/test/rails_dashboard_test.rb +37 -0
  94. data/test/rails_helper_test.rb +36 -0
  95. data/test/rails_test.rb +389 -0
  96. data/test/test_helper.rb +145 -0
  97. data/vanity.gemspec +26 -0
  98. metadata +202 -0
@@ -0,0 +1,157 @@
1
+ module Vanity
2
+ module Adapters
3
+ class << self
4
+ # Creates and returns new MockAdapter.
5
+ #
6
+ # @since 1.4.0
7
+ def mock_connection(spec)
8
+ MockAdapter.new(spec)
9
+ end
10
+ end
11
+
12
+ # Mock adapter. You can use this when running in test environment, staging,
13
+ # wherever you don't care for collecting metrics. Doesn't store anything.
14
+ #
15
+ # @since 1.4.0
16
+ class MockAdapter < AbstractAdapter
17
+ def initialize(options)
18
+ @metrics = @@metrics ||= {}
19
+ @experiments = @@experiments ||= {}
20
+ end
21
+
22
+ def active?
23
+ !!@metrics
24
+ end
25
+
26
+ def disconnect!
27
+ @metrics = nil
28
+ @experiments = nil
29
+ end
30
+
31
+ def reconnect!
32
+ @metrics = @@metrics
33
+ @experiments = @@experiments
34
+ end
35
+
36
+ def to_s
37
+ "mock:/"
38
+ end
39
+
40
+ def flushdb
41
+ @metrics.clear
42
+ @experiments.clear
43
+ end
44
+
45
+
46
+ # -- Metrics --
47
+
48
+ def get_metric_last_update_at(metric)
49
+ @metrics[metric] && @metrics[metric][:last_update_at]
50
+ end
51
+
52
+ def metric_track(metric, timestamp, identity, values)
53
+ @metrics[metric] ||= {}
54
+ current = @metrics[metric][timestamp.to_date] ||= []
55
+ values.each_with_index do |v,i|
56
+ current[i] = (current[i] || 0) + v || 0
57
+ end
58
+ @metrics[metric][:last_update_at] = Time.now
59
+ end
60
+
61
+ def metric_values(metric, from, to)
62
+ hash = @metrics[metric] || {}
63
+ (from.to_date..to.to_date).map { |date| hash[date] || [] }
64
+ end
65
+
66
+ def destroy_metric(metric)
67
+ @metrics.delete metric
68
+ end
69
+
70
+
71
+ # -- Experiments --
72
+
73
+ def set_experiment_created_at(experiment, time)
74
+ @experiments[experiment] ||= {}
75
+ @experiments[experiment][:created_at] ||= time
76
+ end
77
+
78
+ def get_experiment_created_at(experiment)
79
+ @experiments[experiment] && @experiments[experiment][:created_at]
80
+ end
81
+
82
+ def set_experiment_completed_at(experiment, time)
83
+ @experiments[experiment] ||= {}
84
+ @experiments[experiment][:completed_at] ||= time
85
+ end
86
+
87
+ def get_experiment_completed_at(experiment)
88
+ @experiments[experiment] && @experiments[experiment][:completed_at]
89
+ end
90
+
91
+ def is_experiment_completed?(experiment)
92
+ @experiments[experiment] && @experiments[experiment][:completed_at]
93
+ end
94
+
95
+ def ab_counts(experiment, alternative)
96
+ @experiments[experiment] ||= {}
97
+ @experiments[experiment][:alternatives] ||= {}
98
+ alt = @experiments[experiment][:alternatives][alternative] ||= {}
99
+ { :participants => alt[:participants] ? alt[:participants].size : 0,
100
+ :converted => alt[:converted] ? alt[:converted].size : 0,
101
+ :conversions => alt[:conversions] || 0 }
102
+ end
103
+
104
+ def ab_show(experiment, identity, alternative)
105
+ @experiments[experiment] ||= {}
106
+ @experiments[experiment][:showing] ||= {}
107
+ @experiments[experiment][:showing][identity] = alternative
108
+ end
109
+
110
+ def ab_showing(experiment, identity)
111
+ @experiments[experiment] && @experiments[experiment][:showing] && @experiments[experiment][:showing][identity]
112
+ end
113
+
114
+ def ab_not_showing(experiment, identity)
115
+ @experiments[experiment][:showing].delete identity if @experiments[experiment] && @experiments[experiment][:showing]
116
+ end
117
+
118
+ def ab_add_participant(experiment, alternative, identity)
119
+ @experiments[experiment] ||= {}
120
+ @experiments[experiment][:alternatives] ||= {}
121
+ alt = @experiments[experiment][:alternatives][alternative] ||= {}
122
+ alt[:participants] ||= Set.new
123
+ alt[:participants] << identity
124
+ end
125
+
126
+ def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
127
+ @experiments[experiment] ||= {}
128
+ @experiments[experiment][:alternatives] ||= {}
129
+ alt = @experiments[experiment][:alternatives][alternative] ||= {}
130
+ alt[:participants] ||= Set.new
131
+ alt[:converted] ||= Set.new
132
+ alt[:conversions] ||= 0
133
+ if implicit
134
+ alt[:participants] << identity
135
+ else
136
+ participating = alt[:participants].include?(identity)
137
+ end
138
+ alt[:converted] << identity if implicit || participating
139
+ alt[:conversions] += count
140
+ end
141
+
142
+ def ab_get_outcome(experiment)
143
+ @experiments[experiment] ||= {}
144
+ @experiments[experiment][:outcome]
145
+ end
146
+
147
+ def ab_set_outcome(experiment, alternative = 0)
148
+ @experiments[experiment] ||= {}
149
+ @experiments[experiment][:outcome] = alternative
150
+ end
151
+
152
+ def destroy_experiment(experiment)
153
+ @experiments.delete experiment
154
+ end
155
+ end
156
+ end
157
+ end
@@ -0,0 +1,178 @@
1
+ module Vanity
2
+ module Adapters
3
+ class << self
4
+ # Creates new connection to MongoDB and returns MongoAdapter.
5
+ #
6
+ # @since 1.4.0
7
+ def mongo_connection(spec)
8
+ require "mongo"
9
+ MongodbAdapter.new(spec)
10
+ end
11
+ alias :mongodb_connection :mongo_connection
12
+ end
13
+
14
+ # MongoDB adapter.
15
+ #
16
+ # @since 1.4.0
17
+ class MongodbAdapter < AbstractAdapter
18
+ attr_reader :mongo
19
+
20
+ def initialize(options)
21
+ setup_connection(options)
22
+ @options = options.clone
23
+ @options[:database] ||= (@options[:path] && @options[:path].split("/")[1]) || "vanity"
24
+ connect!
25
+ end
26
+
27
+ def setup_connection(options = {})
28
+ if options[:hosts]
29
+ args = (options[:hosts].map{|host| [host, options[:port]] } << {:connect => false})
30
+ @mongo = Mongo::ReplSetConnection.new(*args)
31
+ else
32
+ @mongo = Mongo::Connection.new(options[:host], options[:port], :connect => false)
33
+ end
34
+ @mongo
35
+ end
36
+
37
+ def active?
38
+ @mongo.connected?
39
+ end
40
+
41
+ def disconnect!
42
+ @mongo.close rescue nil if @mongo
43
+ @metrics, @experiments = nil
44
+ @mongo = nil
45
+ end
46
+
47
+ def reconnect!
48
+ disconnect!
49
+ connect!
50
+ end
51
+
52
+ def connect!
53
+ @mongo ||= setup_connection(@options)
54
+ @mongo.connect
55
+ database = @mongo.db(@options[:database])
56
+ database.authenticate @options[:username], @options[:password], true if @options[:username]
57
+ @metrics = database.collection("vanity.metrics")
58
+ @experiments = database.collection("vanity.experiments")
59
+ @participants = database.collection("vanity.participants")
60
+ @participants.create_index [[:experiment, 1], [:identity, 1]], :unique=>true
61
+ @participants.create_index [[:experiment, 1], [:seen, 1]]
62
+ @participants.create_index [[:experiment, 1], [:converted, 1]]
63
+ @mongo
64
+ end
65
+
66
+ def to_s
67
+ userinfo = @options.values_at(:username, :password).join(":") if @options[:username]
68
+ URI::Generic.build(:scheme=>"mongodb", :userinfo=>userinfo, :host=>(@mongo.host || @options[:host]), :port=>(@mongo.port || @options[:port]), :path=>"/#{@options[:database]}").to_s
69
+ end
70
+
71
+ def flushdb
72
+ @metrics.drop
73
+ @experiments.drop
74
+ @participants.drop
75
+ end
76
+
77
+
78
+ # -- Metrics --
79
+
80
+ def get_metric_last_update_at(metric)
81
+ record = @metrics.find_one(:_id=>metric)
82
+ record && record["last_update_at"]
83
+ end
84
+
85
+ def metric_track(metric, timestamp, identity, values)
86
+ inc = {}
87
+ values.each_with_index do |v,i|
88
+ inc["data.#{timestamp.to_date}.#{i}"] = v
89
+ end
90
+ @metrics.update({ :_id=>metric }, { "$inc"=>inc, "$set"=>{ :last_update_at=>Time.now } }, :upsert=>true)
91
+ end
92
+
93
+ def metric_values(metric, from, to)
94
+ record = @metrics.find_one(:_id=>metric)
95
+ data = record && record["data"] || {}
96
+ (from.to_date..to.to_date).map { |date| (data[date.to_s] || {}).values }
97
+ end
98
+
99
+ def destroy_metric(metric)
100
+ @metrics.remove :_id=>metric
101
+ end
102
+
103
+
104
+ # -- Experiments --
105
+
106
+ def set_experiment_created_at(experiment, time)
107
+ @experiments.insert :_id=>experiment, :created_at=>time
108
+ end
109
+
110
+ def get_experiment_created_at(experiment)
111
+ record = @experiments.find_one({ :_id=>experiment }, { :fields=>[:created_at] })
112
+ record && record["created_at"]
113
+ end
114
+
115
+ def set_experiment_completed_at(experiment, time)
116
+ @experiments.update({ :_id=>experiment }, { "$set"=>{ :completed_at=>time } }, :upsert=>true)
117
+ end
118
+
119
+ def get_experiment_completed_at(experiment)
120
+ record = @experiments.find_one({ :_id=>experiment }, { :fields=>[:completed_at] })
121
+ record && record["completed_at"]
122
+ end
123
+
124
+ def is_experiment_completed?(experiment)
125
+ !!@experiments.find_one(:_id=>experiment, :completed_at=>{ "$exists"=>true })
126
+ end
127
+
128
+ def ab_counts(experiment, alternative)
129
+ record = @experiments.find_one({ :_id=>experiment }, { :fields=>[:conversions] })
130
+ conversions = record && record["conversions"]
131
+ { :participants => @participants.find({ :experiment=>experiment, :seen=>alternative }).count,
132
+ :converted => @participants.find({ :experiment=>experiment, :converted=>alternative }).count,
133
+ :conversions => conversions && conversions[alternative.to_s] || 0 }
134
+ end
135
+
136
+ def ab_show(experiment, identity, alternative)
137
+ @participants.update({ :experiment=>experiment, :identity=>identity }, { "$set"=>{ :show=>alternative } }, :upsert=>true)
138
+ end
139
+
140
+ def ab_showing(experiment, identity)
141
+ participant = @participants.find_one({ :experiment=>experiment, :identity=>identity }, { :fields=>[:show] })
142
+ participant && participant["show"]
143
+ end
144
+
145
+ def ab_not_showing(experiment, identity)
146
+ @participants.update({ :experiment=>experiment, :identity=>identity }, { "$unset"=>:show })
147
+ end
148
+
149
+ def ab_add_participant(experiment, alternative, identity)
150
+ @participants.update({ :experiment=>experiment, :identity=>identity }, { "$push"=>{ :seen=>alternative } }, :upsert=>true)
151
+ end
152
+
153
+ def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
154
+ if implicit
155
+ @participants.update({ :experiment=>experiment, :identity=>identity }, { "$push"=>{ :seen=>alternative } }, :upsert=>true)
156
+ else
157
+ participating = @participants.find_one(:experiment=>experiment, :identity=>identity, :seen=>alternative)
158
+ end
159
+ @participants.update({ :experiment=>experiment, :identity=>identity }, { "$push"=>{ :converted=>alternative } }, :upsert=>true) if implicit || participating
160
+ @experiments.update({ :_id=>experiment }, { "$inc"=>{ "conversions.#{alternative}"=>count } }, :upsert=>true)
161
+ end
162
+
163
+ def ab_get_outcome(experiment)
164
+ experiment = @experiments.find_one({ :_id=>experiment }, { :fields=>[:outcome] })
165
+ experiment && experiment["outcome"]
166
+ end
167
+
168
+ def ab_set_outcome(experiment, alternative = 0)
169
+ @experiments.update({ :_id=>experiment }, { "$set"=>{ :outcome=>alternative } }, :upsert=>true)
170
+ end
171
+
172
+ def destroy_experiment(experiment)
173
+ @experiments.remove :_id=>experiment
174
+ @participants.remove :experiment=>experiment
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,160 @@
1
+ module Vanity
2
+ module Adapters
3
+ class << self
4
+ # Creates new connection to Redis and returns RedisAdapter.
5
+ #
6
+ # @since 1.4.0
7
+ def redis_connection(spec)
8
+ require "redis/namespace"
9
+ RedisAdapter.new(spec)
10
+ end
11
+ end
12
+
13
+ # Redis adapter.
14
+ #
15
+ # @since 1.4.0
16
+ class RedisAdapter < AbstractAdapter
17
+ def initialize(options)
18
+ @options = options.clone
19
+ @options[:db] ||= @options[:database] || (@options[:path] && @options.delete(:path).split("/")[1].to_i)
20
+ @options[:thread_safe] = true
21
+ connect!
22
+ end
23
+
24
+ def active?
25
+ !!@redis
26
+ end
27
+
28
+ def disconnect!
29
+ if redis
30
+ begin
31
+ redis.client.disconnect
32
+ rescue Exception => e
33
+ warn("Error while disconnecting from redis: #{e.message}")
34
+ end
35
+ end
36
+ @redis = nil
37
+ end
38
+
39
+ def reconnect!
40
+ disconnect!
41
+ connect!
42
+ end
43
+
44
+ def connect!
45
+ @redis = @options[:redis] || Redis.new(@options)
46
+ @metrics = Redis::Namespace.new("vanity:metrics", :redis=>@redis)
47
+ @experiments = Redis::Namespace.new("vanity:experiments", :redis=>@redis)
48
+ end
49
+
50
+ def to_s
51
+ redis.id
52
+ end
53
+
54
+ def redis
55
+ @redis
56
+ end
57
+
58
+ def flushdb
59
+ @redis.flushdb
60
+ end
61
+
62
+ # -- Metrics --
63
+
64
+ def get_metric_last_update_at(metric)
65
+ last_update_at = @metrics["#{metric}:last_update_at"]
66
+ last_update_at && Time.at(last_update_at.to_i)
67
+ end
68
+
69
+ def metric_track(metric, timestamp, identity, values)
70
+ values.each_with_index do |v,i|
71
+ @metrics.incrby "#{metric}:#{timestamp.to_date}:value:#{i}", v
72
+ end
73
+ @metrics["#{metric}:last_update_at"] = Time.now.to_i
74
+ end
75
+
76
+ def metric_values(metric, from, to)
77
+ single = @metrics.mget(*(from.to_date..to.to_date).map { |date| "#{metric}:#{date}:value:0" }) || []
78
+ single.map { |v| [v] }
79
+ end
80
+
81
+ def destroy_metric(metric)
82
+ @metrics.del *@metrics.keys("#{metric}:*")
83
+ end
84
+
85
+
86
+ # -- Experiments --
87
+
88
+ def set_experiment_created_at(experiment, time)
89
+ @experiments.setnx "#{experiment}:created_at", time.to_i
90
+ end
91
+
92
+ def get_experiment_created_at(experiment)
93
+ created_at = @experiments["#{experiment}:created_at"]
94
+ created_at && Time.at(created_at.to_i)
95
+ end
96
+
97
+ def set_experiment_completed_at(experiment, time)
98
+ @experiments.setnx "#{experiment}:completed_at", time.to_i
99
+ end
100
+
101
+ def get_experiment_completed_at(experiment)
102
+ completed_at = @experiments["#{experiment}:completed_at"]
103
+ completed_at && Time.at(completed_at.to_i)
104
+ end
105
+
106
+ def is_experiment_completed?(experiment)
107
+ @experiments.exists("#{experiment}:completed_at")
108
+ end
109
+
110
+ def ab_counts(experiment, alternative)
111
+ { :participants => @experiments.scard("#{experiment}:alts:#{alternative}:participants").to_i,
112
+ :converted => @experiments.scard("#{experiment}:alts:#{alternative}:converted").to_i,
113
+ :conversions => @experiments["#{experiment}:alts:#{alternative}:conversions"].to_i }
114
+ end
115
+
116
+ def ab_show(experiment, identity, alternative)
117
+ @experiments["#{experiment}:participant:#{identity}:show"] = alternative
118
+ end
119
+
120
+ def ab_showing(experiment, identity)
121
+ alternative = @experiments["#{experiment}:participant:#{identity}:show"]
122
+ alternative && alternative.to_i
123
+ end
124
+
125
+ def ab_not_showing(experiment, identity)
126
+ @experiments.del "#{experiment}:participant:#{identity}:show"
127
+ end
128
+
129
+ def ab_add_participant(experiment, alternative, identity)
130
+ @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
131
+ end
132
+
133
+ def ab_add_conversion(experiment, alternative, identity, count = 1, implicit = false)
134
+ if implicit
135
+ @experiments.sadd "#{experiment}:alts:#{alternative}:participants", identity
136
+ else
137
+ participating = @experiments.sismember("#{experiment}:alts:#{alternative}:participants", identity)
138
+ end
139
+ @experiments.sadd "#{experiment}:alts:#{alternative}:converted", identity if implicit || participating
140
+ @experiments.incrby "#{experiment}:alts:#{alternative}:conversions", count
141
+ end
142
+
143
+ def ab_get_outcome(experiment)
144
+ alternative = @experiments["#{experiment}:outcome"]
145
+ alternative && alternative.to_i
146
+ end
147
+
148
+ def ab_set_outcome(experiment, alternative = 0)
149
+ @experiments.setnx "#{experiment}:outcome", alternative
150
+ end
151
+
152
+ def destroy_experiment(experiment)
153
+ @experiments.del "#{experiment}:outcome", "#{experiment}:created_at", "#{experiment}:completed_at"
154
+ alternatives = @experiments.keys("#{experiment}:alts:*")
155
+ @experiments.del *alternatives unless alternatives.empty?
156
+ end
157
+
158
+ end
159
+ end
160
+ end