sinmetrics 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/Rakefile +25 -7
- data/lib/sinmetrics.rb +1 -0
- data/lib/sinmetrics/abingo.rb +274 -0
- data/lib/sinmetrics/abingo/alternative.rb +30 -0
- data/lib/sinmetrics/abingo/experiment.rb +103 -0
- data/lib/sinmetrics/abingo/statistics.rb +90 -0
- data/spec/abingo_spec.rb +143 -0
- data/spec/kontagent_spec.rb +23 -0
- data/spec/mixpanel_spec.rb +20 -0
- data/spec/spec_helper.rb +13 -0
- metadata +104 -20
- data/Manifest +0 -6
- data/sinmetrics.gemspec +0 -29
data/.gitignore
ADDED
data/Rakefile
CHANGED
@@ -1,11 +1,29 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'rake'
|
3
|
-
require 'echoe'
|
4
3
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
4
|
+
task :default => :spec
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'spec/rake/spectask'
|
8
|
+
desc "Run all examples"
|
9
|
+
Spec::Rake::SpecTask.new('spec') do |t|
|
10
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
11
|
+
t.spec_opts = ['-cubfs']
|
12
|
+
end
|
13
|
+
|
14
|
+
Spec::Rake::SpecTask.new('spec:rcov') do |t|
|
15
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
16
|
+
t.spec_opts = ['-cfs']
|
17
|
+
t.rcov = true
|
18
|
+
t.rcov_opts = ['--exclude', 'gems,spec/,examples/']
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'spec/rake/verify_rcov'
|
22
|
+
RCov::VerifyTask.new(:verify_rcov => 'spec:rcov') do |t|
|
23
|
+
t.threshold = 77.0
|
24
|
+
t.require_exact_threshold = false
|
25
|
+
end
|
26
|
+
|
27
|
+
rescue LoadError
|
28
|
+
puts "spec targets require RSpec"
|
11
29
|
end
|
data/lib/sinmetrics.rb
CHANGED
@@ -0,0 +1,274 @@
|
|
1
|
+
#This class is outside code's main interface into the ABingo A/B testing framework.
|
2
|
+
#Unless you're fiddling with implementation details, it is the only one you need worry about.
|
3
|
+
|
4
|
+
#Usage of ABingo, including practical hints, is covered at http://www.bingocardcreator.com/abingo
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'sinatra/base'
|
8
|
+
require 'active_support'
|
9
|
+
rescue LoadError
|
10
|
+
retry if require 'rubygems'
|
11
|
+
raise
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
module Sinatra
|
16
|
+
class AbingoObject
|
17
|
+
|
18
|
+
#Defined options:
|
19
|
+
# :enable_specification => if true, allow params[test_name] to override the calculated value for a test.
|
20
|
+
def initialize app={}
|
21
|
+
if app.respond_to?(:options)
|
22
|
+
@app = app
|
23
|
+
[:identity, :enable_specification, :cache, :salt].each do |var|
|
24
|
+
begin
|
25
|
+
instance_variable_set("@#{var}", app.options.send("abingo_#{var}"))
|
26
|
+
rescue NoMethodError
|
27
|
+
end
|
28
|
+
end
|
29
|
+
else
|
30
|
+
[:identity, :enable_specification, :cache, :salt].each do |var|
|
31
|
+
instance_variable_set("@#{var}", app[var]) if app.has_key?(var)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
@identity ||= rand(10 ** 10).to_i.to_s
|
35
|
+
@cache ||= ActiveSupport::Cache::MemoryStore.new
|
36
|
+
@salt ||= ''
|
37
|
+
end
|
38
|
+
|
39
|
+
attr_reader :app, :enable_specification, :cache, :salt
|
40
|
+
attr_accessor :identity
|
41
|
+
|
42
|
+
#A simple convenience method for doing an A/B test. Returns true or false.
|
43
|
+
#If you pass it a block, it will bind the choice to the variable given to the block.
|
44
|
+
def flip(test_name)
|
45
|
+
if block_given?
|
46
|
+
yield(test(test_name, [true, false]))
|
47
|
+
else
|
48
|
+
test(test_name, [true, false])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
#This is the meat of A/Bingo.
|
53
|
+
#options accepts
|
54
|
+
# :multiple_participation (true or false)
|
55
|
+
# :conversion name of conversion to listen for (alias: conversion_name)
|
56
|
+
def test(test_name, alternatives, options = {})
|
57
|
+
short_circuit = cache.read("Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_"))
|
58
|
+
unless short_circuit.nil?
|
59
|
+
return short_circuit #Test has been stopped, pick canonical alternative.
|
60
|
+
end
|
61
|
+
|
62
|
+
unless Experiment.exists?(self, test_name)
|
63
|
+
conversion_name = options[:conversion] || options[:conversion_name]
|
64
|
+
Experiment.start_experiment!(self, test_name, parse_alternatives(alternatives), conversion_name)
|
65
|
+
end
|
66
|
+
|
67
|
+
choice = find_alternative_for_user(test_name, alternatives)
|
68
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
69
|
+
|
70
|
+
#Set this user to participate in this experiment, and increment participants count.
|
71
|
+
if options[:multiple_participation] || !(participating_tests.include?(test_name))
|
72
|
+
participating_tests = participating_tests.dup if participating_tests.frozen?
|
73
|
+
unless participating_tests.include?(test_name)
|
74
|
+
participating_tests << test_name
|
75
|
+
cache.write("Abingo::participating_tests::#{identity}", participating_tests)
|
76
|
+
end
|
77
|
+
Alternative.score_participation(self, test_name)
|
78
|
+
end
|
79
|
+
|
80
|
+
if block_given?
|
81
|
+
yield(choice)
|
82
|
+
else
|
83
|
+
choice
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
#Scores conversions for tests.
|
88
|
+
#test_name_or_array supports three types of input:
|
89
|
+
#
|
90
|
+
#A conversion name: scores a conversion for any test the user is participating in which
|
91
|
+
# is listening to the specified conversion.
|
92
|
+
#
|
93
|
+
#A test name: scores a conversion for the named test if the user is participating in it.
|
94
|
+
#
|
95
|
+
#An array of either of the above: for each element of the array, process as above.
|
96
|
+
#
|
97
|
+
#nil: score a conversion for every test the u
|
98
|
+
def bingo!(name = nil, options = {})
|
99
|
+
if name.kind_of? Array
|
100
|
+
name.map do |single_test|
|
101
|
+
self.bingo!(single_test, options)
|
102
|
+
end
|
103
|
+
else
|
104
|
+
if name.nil?
|
105
|
+
#Score all participating tests
|
106
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
107
|
+
participating_tests.each do |participating_test|
|
108
|
+
self.bingo!(participating_test, options)
|
109
|
+
end
|
110
|
+
else #Could be a test name or conversion name.
|
111
|
+
conversion_name = name.gsub(" ", "_")
|
112
|
+
tests_listening_to_conversion = cache.read("Abingo::tests_listening_to_conversion#{conversion_name}")
|
113
|
+
if tests_listening_to_conversion
|
114
|
+
if tests_listening_to_conversion.size > 1
|
115
|
+
tests_listening_to_conversion.map do |individual_test|
|
116
|
+
self.score_conversion!(individual_test.to_s, options)
|
117
|
+
end
|
118
|
+
elsif tests_listening_to_conversion.size == 1
|
119
|
+
test_name_str = tests_listening_to_conversion.first.to_s
|
120
|
+
self.score_conversion!(test_name_str, options)
|
121
|
+
end
|
122
|
+
else
|
123
|
+
#No tests listening for this conversion. Assume it is just a test name.
|
124
|
+
test_name_str = name.to_s
|
125
|
+
self.score_conversion!(test_name_str, options)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
#protected
|
132
|
+
|
133
|
+
#For programmer convenience, we allow you to specify what the alternatives for
|
134
|
+
#an experiment are in a few ways. Thus, we need to actually be able to handle
|
135
|
+
#all of them. We fire this parser very infrequently (once per test, typically)
|
136
|
+
#so it can be as complicated as we want.
|
137
|
+
# Integer => a number 1 through N
|
138
|
+
# Range => a number within the range
|
139
|
+
# Array => an element of the array.
|
140
|
+
# Hash => assumes a hash of something to int. We pick one of the
|
141
|
+
# somethings, weighted accorded to the ints provided. e.g.
|
142
|
+
# {:a => 2, :b => 3} produces :a 40% of the time, :b 60%.
|
143
|
+
#
|
144
|
+
#Alternatives are always represented internally as an array.
|
145
|
+
def parse_alternatives(alternatives)
|
146
|
+
if alternatives.kind_of? Array
|
147
|
+
return alternatives
|
148
|
+
elsif alternatives.kind_of? Integer
|
149
|
+
return (1..alternatives).to_a
|
150
|
+
elsif alternatives.kind_of? Range
|
151
|
+
return alternatives.to_a
|
152
|
+
elsif alternatives.kind_of? Hash
|
153
|
+
alternatives_array = []
|
154
|
+
alternatives.each do |key, value|
|
155
|
+
if value.kind_of? Integer
|
156
|
+
alternatives_array += [key] * value
|
157
|
+
else
|
158
|
+
raise "You gave a hash with #{key} => #{value} as an element. The value must be an integral weight."
|
159
|
+
end
|
160
|
+
end
|
161
|
+
return alternatives_array
|
162
|
+
else
|
163
|
+
raise "I don't know how to turn [#{alternatives}] into an array of alternatives."
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def retrieve_alternatives(test_name, alternatives)
|
168
|
+
cache_key = "Abingo::Experiment::#{test_name}::alternatives".gsub(" ","_")
|
169
|
+
alternative_array = self.cache.fetch(cache_key) do
|
170
|
+
self.parse_alternatives(alternatives)
|
171
|
+
end
|
172
|
+
alternative_array
|
173
|
+
end
|
174
|
+
|
175
|
+
def find_alternative_for_user(test_name, alternatives)
|
176
|
+
alternatives_array = retrieve_alternatives(test_name, alternatives)
|
177
|
+
alternatives_array[self.modulo_choice(test_name, alternatives_array.size)]
|
178
|
+
end
|
179
|
+
|
180
|
+
#Quickly determines what alternative to show a given user. Given a test name
|
181
|
+
#and their identity, we hash them together (which, for MD5, provably introduces
|
182
|
+
#enough entropy that we don't care) otherwise
|
183
|
+
def modulo_choice(test_name, choices_count)
|
184
|
+
Digest::MD5.hexdigest(salt.to_s + test_name + self.identity.to_s).to_i(16) % choices_count
|
185
|
+
end
|
186
|
+
|
187
|
+
def score_conversion!(test_name, options = {})
|
188
|
+
test_name.gsub!(" ", "_")
|
189
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
190
|
+
if options[:assume_participation] || participating_tests.include?(test_name)
|
191
|
+
cache_key = "Abingo::conversions(#{identity},#{test_name}"
|
192
|
+
if options[:multiple_conversions] || !cache.read(cache_key)
|
193
|
+
Alternative.score_conversion(self, test_name)
|
194
|
+
if cache.exist?(cache_key)
|
195
|
+
cache.increment(cache_key)
|
196
|
+
else
|
197
|
+
cache.write(cache_key, 1)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
module AbingoObject::ConversionRate
|
205
|
+
def conversion_rate
|
206
|
+
1.0 * conversions / participants
|
207
|
+
end
|
208
|
+
|
209
|
+
def pretty_conversion_rate
|
210
|
+
sprintf("%4.2f%%", conversion_rate * 100)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
module AbingoHelper
|
215
|
+
def abingo
|
216
|
+
env['abingo.helper'] ||= AbingoObject.new(self)
|
217
|
+
end
|
218
|
+
|
219
|
+
def ab_test(test_name, alternatives = nil, options = {})
|
220
|
+
if (abingo.enable_specification && !params[test_name].blank?)
|
221
|
+
choice = params[test_name]
|
222
|
+
elsif (alternatives.nil?)
|
223
|
+
choice = abingo.flip(test_name)
|
224
|
+
else
|
225
|
+
choice = abingo.test(test_name, alternatives, options)
|
226
|
+
end
|
227
|
+
|
228
|
+
if block_given?
|
229
|
+
yield(choice)
|
230
|
+
else
|
231
|
+
choice
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
alias ab abingo
|
236
|
+
end
|
237
|
+
|
238
|
+
class AbingoSettings
|
239
|
+
def initialize app, &blk
|
240
|
+
@app = app
|
241
|
+
instance_eval &blk
|
242
|
+
end
|
243
|
+
%w[identity backend enable_specification cache salt].each do |param|
|
244
|
+
class_eval %[
|
245
|
+
def #{param} val, &blk
|
246
|
+
@app.set :abingo_#{param}, val
|
247
|
+
end
|
248
|
+
]
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
module Abingo
|
253
|
+
def abingo &blk
|
254
|
+
AbingoSettings.new(self, &blk)
|
255
|
+
end
|
256
|
+
|
257
|
+
def self.registered app
|
258
|
+
app.helpers AbingoHelper
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
Application.register Abingo
|
263
|
+
end
|
264
|
+
|
265
|
+
Abingo = Sinatra::AbingoObject
|
266
|
+
|
267
|
+
require 'dm-core'
|
268
|
+
require 'dm-aggregates'
|
269
|
+
require 'dm-observer'
|
270
|
+
require 'dm-timestamps'
|
271
|
+
require 'dm-adjust'
|
272
|
+
require 'abingo/statistics'
|
273
|
+
require 'abingo/alternative'
|
274
|
+
require 'abingo/experiment'
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class Abingo::Alternative
|
2
|
+
include Abingo::ConversionRate
|
3
|
+
include DataMapper::Resource
|
4
|
+
|
5
|
+
property :id, Serial
|
6
|
+
property :content, String
|
7
|
+
property :lookup, String, :length => 32, :index => true
|
8
|
+
property :weight, Integer, :default => 1
|
9
|
+
property :participants, Integer, :default => 0
|
10
|
+
property :conversions, Integer, :default => 0
|
11
|
+
|
12
|
+
belongs_to :experiment
|
13
|
+
|
14
|
+
def self.calculate_lookup(abingo, test_name, alternative_name)
|
15
|
+
Digest::MD5.hexdigest(abingo.salt + test_name + alternative_name.to_s)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.score_conversion(abingo, test_name)
|
19
|
+
viewed_alternative = abingo.find_alternative_for_user(test_name,
|
20
|
+
Abingo::Experiment.alternatives_for_test(abingo, test_name))
|
21
|
+
all(:lookup => self.calculate_lookup(abingo, test_name, viewed_alternative)).adjust!(:conversions => 1)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.score_participation(abingo, test_name)
|
25
|
+
viewed_alternative = abingo.find_alternative_for_user(test_name,
|
26
|
+
Abingo::Experiment.alternatives_for_test(abingo, test_name))
|
27
|
+
all(:lookup => self.calculate_lookup(abingo, test_name, viewed_alternative)).adjust!(:participants => 1)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
class Abingo::Experiment
|
2
|
+
include DataMapper::Resource
|
3
|
+
include Abingo::Statistics
|
4
|
+
include Abingo::ConversionRate
|
5
|
+
|
6
|
+
property :test_name, String, :key => true
|
7
|
+
property :status, String
|
8
|
+
|
9
|
+
has n, :alternatives, "Alternative"
|
10
|
+
|
11
|
+
def cache_keys
|
12
|
+
["Abingo::Experiment::exists(#{test_name})".gsub(" ", "_"),
|
13
|
+
"Abingo::Experiment::#{test_name}::alternatives".gsub(" ","_"),
|
14
|
+
"Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_")
|
15
|
+
]
|
16
|
+
end
|
17
|
+
|
18
|
+
def before_destroy
|
19
|
+
cache_keys.each do |key|
|
20
|
+
Abingo.cache.delete key
|
21
|
+
end
|
22
|
+
true
|
23
|
+
end
|
24
|
+
|
25
|
+
def participants
|
26
|
+
alternatives.sum(:participants)
|
27
|
+
end
|
28
|
+
|
29
|
+
def conversions
|
30
|
+
alternatives.sum(:conversions)
|
31
|
+
end
|
32
|
+
|
33
|
+
def best_alternative
|
34
|
+
alternatives.max do |a,b|
|
35
|
+
a.conversion_rate <=> b.conversion_rate
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.exists?(abingo, test_name)
|
40
|
+
cache_key = "Abingo::Experiment::exists(#{test_name})".gsub(" ", "_")
|
41
|
+
ret = abingo.cache.fetch(cache_key) do
|
42
|
+
count = Abingo::Experiment.count(:conditions => {:test_name => test_name})
|
43
|
+
count > 0 ? count : nil
|
44
|
+
end
|
45
|
+
(!ret.nil?)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.alternatives_for_test(abingo, test_name)
|
49
|
+
cache_key = "Abingo::#{test_name}::alternatives".gsub(" ","_")
|
50
|
+
abingo.cache.fetch(cache_key) do
|
51
|
+
experiment = Abingo::Experiment.first(:test_name => test_name)
|
52
|
+
alternatives_array = abingo.cache.fetch(cache_key) do
|
53
|
+
tmp_array = experiment.alternatives.map do |alt|
|
54
|
+
[alt.content, alt.weight]
|
55
|
+
end
|
56
|
+
tmp_hash = tmp_array.inject({}) {|hash, couplet| hash[couplet[0]] = couplet[1]; hash}
|
57
|
+
abingo.parse_alternatives(tmp_hash)
|
58
|
+
end
|
59
|
+
alternatives_array
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.start_experiment!(abingo, test_name, alternatives_array, conversion_name = nil)
|
64
|
+
conversion_name ||= test_name
|
65
|
+
conversion_name.gsub!(" ", "_")
|
66
|
+
cloned_alternatives_array = alternatives_array.clone
|
67
|
+
Abingo::Experiment.transaction do |txn|
|
68
|
+
experiment = Abingo::Experiment.first_or_create(:test_name => test_name)
|
69
|
+
experiment.alternatives.destroy #Blows away alternatives for pre-existing experiments.
|
70
|
+
while (cloned_alternatives_array.size > 0)
|
71
|
+
alt = cloned_alternatives_array[0]
|
72
|
+
weight = cloned_alternatives_array.size - (cloned_alternatives_array - [alt]).size
|
73
|
+
experiment.alternatives << Abingo::Alternative.new(:content => alt, :weight => weight,
|
74
|
+
:lookup => Abingo::Alternative.calculate_lookup(abingo, test_name, alt))
|
75
|
+
cloned_alternatives_array -= [alt]
|
76
|
+
end
|
77
|
+
experiment.status = "Live"
|
78
|
+
experiment.save
|
79
|
+
abingo.cache.write("Abingo::Experiment::exists(#{test_name})".gsub(" ", "_"), 1)
|
80
|
+
|
81
|
+
#This might have issues in very, very high concurrency environments...
|
82
|
+
|
83
|
+
tests_listening_to_conversion = abingo.cache.read("Abingo::tests_listening_to_conversion#{conversion_name}") || []
|
84
|
+
tests_listening_to_conversion = tests_listening_to_conversion.dup if tests_listening_to_conversion.frozen?
|
85
|
+
tests_listening_to_conversion << test_name unless tests_listening_to_conversion.include? test_name
|
86
|
+
abingo.cache.write("Abingo::tests_listening_to_conversion#{conversion_name}", tests_listening_to_conversion)
|
87
|
+
experiment
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def end_experiment!(abingo, final_alternative, conversion_name = nil)
|
92
|
+
conversion_name ||= test_name
|
93
|
+
Abingo::Experiment.transaction do
|
94
|
+
alternatives.each do |alternative|
|
95
|
+
alternative.lookup = "Experiment completed. #{alternative.id}"
|
96
|
+
alternative.save!
|
97
|
+
end
|
98
|
+
update(:status => "Finished")
|
99
|
+
abingo.cache.write("Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_"), final_alternative)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
#Designed to be included into Abingo::Experiment, but you can feel free to adapt this
|
2
|
+
#to anything you want.
|
3
|
+
|
4
|
+
module Abingo::Statistics
|
5
|
+
|
6
|
+
HANDY_Z_SCORE_CHEATSHEET = [[0.10, 1.29], [0.05, 1.65], [0.01, 2.33], [0.001, 3.08]]
|
7
|
+
|
8
|
+
PERCENTAGES = {0.10 => '90%', 0.05 => '95%', 0.01 => '99%', 0.001 => '99.9%'}
|
9
|
+
|
10
|
+
DESCRIPTION_IN_WORDS = {0.10 => 'fairly confident', 0.05 => 'confident',
|
11
|
+
0.01 => 'very confident', 0.001 => 'extremely confident'}
|
12
|
+
def zscore
|
13
|
+
if alternatives.size != 2
|
14
|
+
raise "Sorry, can't currently automatically calculate statistics for A/B tests with > 2 alternatives."
|
15
|
+
end
|
16
|
+
|
17
|
+
if (alternatives[0].participants == 0) || (alternatives[1].participants == 0)
|
18
|
+
raise "Can't calculate the z score if either of the alternatives lacks participants."
|
19
|
+
end
|
20
|
+
|
21
|
+
cr1 = alternatives[0].conversion_rate
|
22
|
+
cr2 = alternatives[1].conversion_rate
|
23
|
+
|
24
|
+
n1 = alternatives[0].participants
|
25
|
+
n2 = alternatives[1].participants
|
26
|
+
|
27
|
+
numerator = cr1 - cr2
|
28
|
+
frac1 = cr1 * (1 - cr1) / n1
|
29
|
+
frac2 = cr2 * (1 - cr2) / n2
|
30
|
+
|
31
|
+
numerator / ((frac1 + frac2) ** 0.5)
|
32
|
+
end
|
33
|
+
|
34
|
+
def p_value
|
35
|
+
index = 0
|
36
|
+
z = zscore
|
37
|
+
z = z.abs
|
38
|
+
found_p = nil
|
39
|
+
while index < HANDY_Z_SCORE_CHEATSHEET.size do
|
40
|
+
if (z > HANDY_Z_SCORE_CHEATSHEET[index][1])
|
41
|
+
found_p = HANDY_Z_SCORE_CHEATSHEET[index][0]
|
42
|
+
end
|
43
|
+
index += 1
|
44
|
+
end
|
45
|
+
found_p
|
46
|
+
end
|
47
|
+
|
48
|
+
def is_statistically_significant?(p = 0.05)
|
49
|
+
p_value <= p
|
50
|
+
end
|
51
|
+
|
52
|
+
def pretty_conversion_rate
|
53
|
+
sprintf("%4.2f%%", conversion_rate * 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
def describe_result_in_words
|
57
|
+
begin
|
58
|
+
z = zscore
|
59
|
+
rescue
|
60
|
+
return "Could not execute the significance test because one or more of the alternatives has not been seen yet."
|
61
|
+
end
|
62
|
+
p = p_value
|
63
|
+
|
64
|
+
words = ""
|
65
|
+
if (alternatives[0].participants < 10) || (alternatives[1].participants < 10)
|
66
|
+
words += "Take these results with a grain of salt since your samples are so small: "
|
67
|
+
end
|
68
|
+
|
69
|
+
alts = alternatives - [best_alternative]
|
70
|
+
worst_alternative = alts.first
|
71
|
+
|
72
|
+
words += "The best alternative you have is: [#{best_alternative.content}], which had "
|
73
|
+
words += "#{best_alternative.conversions} conversions from #{best_alternative.participants} participants "
|
74
|
+
words += "(#{best_alternative.pretty_conversion_rate}). The other alternative was [#{worst_alternative.content}], "
|
75
|
+
words += "which had #{worst_alternative.conversions} conversions from #{worst_alternative.participants} participants "
|
76
|
+
words += "(#{worst_alternative.pretty_conversion_rate}). "
|
77
|
+
|
78
|
+
if (p.nil?)
|
79
|
+
words += "However, this difference is not statistically significant."
|
80
|
+
else
|
81
|
+
words += "This difference is #{PERCENTAGES[p]} likely to be statistically significant, which means you can be "
|
82
|
+
words += "#{DESCRIPTION_IN_WORDS[p]} that it is the result of your alternatives actually mattering, rather than "
|
83
|
+
words += "being due to random chance. However, this statistical test can't measure how likely the currently "
|
84
|
+
words += "observed magnitude of the difference is to be accurate or not. It only says \"better\", not \"better "
|
85
|
+
words += "by so much\"."
|
86
|
+
end
|
87
|
+
words
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
data/spec/abingo_spec.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Abingo' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
ab = Abingo.new :identity => 'abcd'
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class AbingoApp < Sinatra::Base
|
11
|
+
register Sinatra::Abingo
|
12
|
+
abingo do
|
13
|
+
identity 'abcd'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
app = AbingoApp.new
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'Abingo Specs' do
|
23
|
+
before :each do
|
24
|
+
@abingo = Abingo.new
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should automatically assign identity" do
|
28
|
+
@abingo.identity.should_not be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should parse alternatives" do
|
32
|
+
array = %w{a b c}
|
33
|
+
@abingo.parse_alternatives(array).should == array
|
34
|
+
@abingo.parse_alternatives(65).size.should == 65
|
35
|
+
@abingo.parse_alternatives(2..5).size.should == 4
|
36
|
+
@abingo.parse_alternatives(2..5).should_not include(1)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should create experiments" do
|
40
|
+
Abingo::Experiment.count.should == 0
|
41
|
+
Abingo::Alternative.count.should == 0
|
42
|
+
alternatives = %w{A B}
|
43
|
+
alternative_selected = @abingo.test("unit_test_sample_A", alternatives)
|
44
|
+
Abingo::Experiment.count.should == 1
|
45
|
+
Abingo::Alternative.count.should == 2
|
46
|
+
alternatives.should include(alternative_selected)
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should be able to call exists" do
|
50
|
+
@abingo.test("exist words right", %w{does does_not})
|
51
|
+
Abingo::Experiment.exists?(@abingo, "exist words right").should be_true
|
52
|
+
Abingo::Experiment.exists?(@abingo, "other words right").should_not be_true
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should pick alternatives consistently" do
|
56
|
+
alternative_picked = @abingo.test("consistency_test", 1..100)
|
57
|
+
100.times do
|
58
|
+
@abingo.test("consistency_test", 1..100).should == alternative_picked
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should have working participation" do
|
63
|
+
new_tests = %w{participationA participationB participationC}
|
64
|
+
new_tests.map do |test_name|
|
65
|
+
@abingo.test(test_name, 1..5)
|
66
|
+
end
|
67
|
+
|
68
|
+
participating_tests = @abingo.cache.read("Abingo::participating_tests::#{@abingo.identity}") || []
|
69
|
+
|
70
|
+
new_tests.map do |test_name|
|
71
|
+
participating_tests.should include(test_name)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should count participants" do
|
76
|
+
test_name = "participants_counted_test"
|
77
|
+
alternative = @abingo.test(test_name, %w{a b c})
|
78
|
+
|
79
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
80
|
+
lookup = Abingo::Alternative.calculate_lookup(@abingo, test_name, alternative)
|
81
|
+
chosen_alt = Abingo::Alternative.first(:lookup => lookup)
|
82
|
+
ex.participants.should == 1
|
83
|
+
chosen_alt.participants.should == 1
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should track conversions by test name" do
|
87
|
+
test_name = "conversion_test_by_name"
|
88
|
+
alternative = @abingo.test(test_name, %w{a b c})
|
89
|
+
@abingo.bingo!(test_name)
|
90
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
91
|
+
lookup = Abingo::Alternative.calculate_lookup(@abingo, test_name, alternative)
|
92
|
+
chosen_alt = Abingo::Alternative.first(:lookup => lookup)
|
93
|
+
ex.conversions.should == 1
|
94
|
+
chosen_alt.conversions.should == 1
|
95
|
+
|
96
|
+
@abingo.bingo!(test_name)
|
97
|
+
|
98
|
+
#Should still only have one because this conversion should not be double counted.
|
99
|
+
#We haven't specified that in the test options.
|
100
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
101
|
+
ex.conversions.should == 1
|
102
|
+
end
|
103
|
+
|
104
|
+
it "should track conversions by conversion name" do
|
105
|
+
conversion_name = "purchase"
|
106
|
+
tests = %w{conversionTrackingByConversionNameA conversionTrackingByConversionNameB conversionTrackingByConversionNameC}
|
107
|
+
tests.map do |test_name|
|
108
|
+
@abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
109
|
+
end
|
110
|
+
|
111
|
+
@abingo.bingo!(conversion_name)
|
112
|
+
tests.map do |test_name|
|
113
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
114
|
+
ex.conversions.should == 1
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should be possible to short circuit tests" do
|
119
|
+
conversion_name = "purchase"
|
120
|
+
test_name = "short circuit test"
|
121
|
+
alt_picked = @abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
122
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
123
|
+
alt_not_picked = (%w{A B} - [alt_picked]).first
|
124
|
+
|
125
|
+
ex.end_experiment!(@abingo, alt_not_picked, conversion_name)
|
126
|
+
|
127
|
+
ex.reload
|
128
|
+
ex.status.should == "Finished"
|
129
|
+
|
130
|
+
@abingo.bingo!(test_name) #Should not be counted, test is over.
|
131
|
+
ex.conversions.should == 0
|
132
|
+
|
133
|
+
old_identity = @abingo.identity
|
134
|
+
@abingo.identity = "shortCircuitTestNewIdentity"
|
135
|
+
@abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
136
|
+
@abingo.identity = old_identity
|
137
|
+
ex.reload
|
138
|
+
|
139
|
+
# Original identity counted, new identity not counted b/c test stopped
|
140
|
+
ex.participants.should == 1
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Kontagent' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
kontagent = Kontagent.new :api_key => '1234', :secret => '5678'
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class KontagentApp < Sinatra::Base
|
11
|
+
register Sinatra::Kontagent
|
12
|
+
kontagent do
|
13
|
+
api_key "1234"
|
14
|
+
secret "5678"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
app = KontagentApp.new
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Mixpanel' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
mp = Mixpanel.new :token => "abcdefghijklmnopqrstuvwyxz"
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class MixpanelApp < Sinatra::Base
|
11
|
+
register Sinatra::Mixpanel
|
12
|
+
mixpanel do
|
13
|
+
token "abcdefghijklmnopqrstuvwyxz"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
app = MixpanelApp.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'spec'
|
5
|
+
require 'sinmetrics'
|
6
|
+
|
7
|
+
# establish in-memory database for testing
|
8
|
+
DataMapper.setup(:default, "sqlite3::memory:")
|
9
|
+
|
10
|
+
Spec::Runner.configure do |config|
|
11
|
+
# reset database before each example is run
|
12
|
+
config.before(:each) { DataMapper.auto_migrate! }
|
13
|
+
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 5
|
9
|
+
version: 0.0.5
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Luke Petre
|
@@ -14,11 +14,94 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-04-
|
17
|
+
date: 2010-04-14 00:00:00 +01:00
|
18
18
|
default_executable:
|
19
|
-
dependencies:
|
20
|
-
|
21
|
-
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: dm-core
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
- 10
|
30
|
+
- 2
|
31
|
+
version: 0.10.2
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: dm-aggregates
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 0
|
43
|
+
- 10
|
44
|
+
- 2
|
45
|
+
version: 0.10.2
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: dm-observer
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 0
|
57
|
+
- 10
|
58
|
+
- 2
|
59
|
+
version: 0.10.2
|
60
|
+
type: :runtime
|
61
|
+
version_requirements: *id003
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: dm-timestamps
|
64
|
+
prerelease: false
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
segments:
|
70
|
+
- 0
|
71
|
+
- 10
|
72
|
+
- 2
|
73
|
+
version: 0.10.2
|
74
|
+
type: :runtime
|
75
|
+
version_requirements: *id004
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: dm-adjust
|
78
|
+
prerelease: false
|
79
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
segments:
|
84
|
+
- 0
|
85
|
+
- 10
|
86
|
+
- 2
|
87
|
+
version: 0.10.2
|
88
|
+
type: :runtime
|
89
|
+
version_requirements: *id005
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: activesupport
|
92
|
+
prerelease: false
|
93
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ~>
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
segments:
|
98
|
+
- 2
|
99
|
+
- 3
|
100
|
+
- 5
|
101
|
+
version: 2.3.5
|
102
|
+
type: :runtime
|
103
|
+
version_requirements: *id006
|
104
|
+
description: A full-featured metrics extension for the sinatra webapp framework
|
22
105
|
email: lpetre@gmail.com
|
23
106
|
executables: []
|
24
107
|
|
@@ -27,25 +110,27 @@ extensions: []
|
|
27
110
|
extra_rdoc_files: []
|
28
111
|
|
29
112
|
files:
|
30
|
-
-
|
113
|
+
- .gitignore
|
31
114
|
- README
|
32
115
|
- Rakefile
|
33
116
|
- lib/sinmetrics.rb
|
117
|
+
- lib/sinmetrics/abingo.rb
|
118
|
+
- lib/sinmetrics/abingo/alternative.rb
|
119
|
+
- lib/sinmetrics/abingo/experiment.rb
|
120
|
+
- lib/sinmetrics/abingo/statistics.rb
|
34
121
|
- lib/sinmetrics/kontagent.rb
|
35
122
|
- lib/sinmetrics/mixpanel.rb
|
36
|
-
-
|
123
|
+
- spec/abingo_spec.rb
|
124
|
+
- spec/kontagent_spec.rb
|
125
|
+
- spec/mixpanel_spec.rb
|
126
|
+
- spec/spec_helper.rb
|
37
127
|
has_rdoc: true
|
38
128
|
homepage: http://github.com/lpetre/sinmetrics
|
39
129
|
licenses: []
|
40
130
|
|
41
131
|
post_install_message:
|
42
|
-
rdoc_options:
|
43
|
-
|
44
|
-
- --inline-source
|
45
|
-
- --title
|
46
|
-
- Sinmetrics
|
47
|
-
- --main
|
48
|
-
- README
|
132
|
+
rdoc_options: []
|
133
|
+
|
49
134
|
require_paths:
|
50
135
|
- lib
|
51
136
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -60,15 +145,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
60
145
|
- - ">="
|
61
146
|
- !ruby/object:Gem::Version
|
62
147
|
segments:
|
63
|
-
-
|
64
|
-
|
65
|
-
version: "1.2"
|
148
|
+
- 0
|
149
|
+
version: "0"
|
66
150
|
requirements: []
|
67
151
|
|
68
|
-
rubyforge_project:
|
152
|
+
rubyforge_project:
|
69
153
|
rubygems_version: 1.3.6
|
70
154
|
signing_key:
|
71
155
|
specification_version: 3
|
72
|
-
summary:
|
156
|
+
summary: simple sinatra metrics extension
|
73
157
|
test_files: []
|
74
158
|
|
data/Manifest
DELETED
data/sinmetrics.gemspec
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
# -*- encoding: utf-8 -*-
|
2
|
-
|
3
|
-
Gem::Specification.new do |s|
|
4
|
-
s.name = %q{sinmetrics}
|
5
|
-
s.version = "0.0.4"
|
6
|
-
|
7
|
-
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
|
-
s.authors = ["Luke Petre"]
|
9
|
-
s.date = %q{2010-04-05}
|
10
|
-
s.description = %q{Some metrics helpers for the Sinatra web framework}
|
11
|
-
s.email = %q{lpetre@gmail.com}
|
12
|
-
s.files = ["Manifest", "README", "Rakefile", "lib/sinmetrics.rb", "lib/sinmetrics/kontagent.rb", "lib/sinmetrics/mixpanel.rb", "sinmetrics.gemspec"]
|
13
|
-
s.homepage = %q{http://github.com/lpetre/sinmetrics}
|
14
|
-
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Sinmetrics", "--main", "README"]
|
15
|
-
s.require_paths = ["lib"]
|
16
|
-
s.rubyforge_project = %q{sinmetrics}
|
17
|
-
s.rubygems_version = %q{1.3.6}
|
18
|
-
s.summary = %q{Some metrics helpers for the Sinatra web framework}
|
19
|
-
|
20
|
-
if s.respond_to? :specification_version then
|
21
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
22
|
-
s.specification_version = 3
|
23
|
-
|
24
|
-
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
25
|
-
else
|
26
|
-
end
|
27
|
-
else
|
28
|
-
end
|
29
|
-
end
|