sinmetrics 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/Rakefile +25 -7
- data/lib/sinmetrics.rb +1 -0
- data/lib/sinmetrics/abingo.rb +274 -0
- data/lib/sinmetrics/abingo/alternative.rb +30 -0
- data/lib/sinmetrics/abingo/experiment.rb +103 -0
- data/lib/sinmetrics/abingo/statistics.rb +90 -0
- data/spec/abingo_spec.rb +143 -0
- data/spec/kontagent_spec.rb +23 -0
- data/spec/mixpanel_spec.rb +20 -0
- data/spec/spec_helper.rb +13 -0
- metadata +104 -20
- data/Manifest +0 -6
- data/sinmetrics.gemspec +0 -29
data/.gitignore
ADDED
data/Rakefile
CHANGED
@@ -1,11 +1,29 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'rake'
|
3
|
-
require 'echoe'
|
4
3
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
4
|
+
task :default => :spec
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'spec/rake/spectask'
|
8
|
+
desc "Run all examples"
|
9
|
+
Spec::Rake::SpecTask.new('spec') do |t|
|
10
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
11
|
+
t.spec_opts = ['-cubfs']
|
12
|
+
end
|
13
|
+
|
14
|
+
Spec::Rake::SpecTask.new('spec:rcov') do |t|
|
15
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
16
|
+
t.spec_opts = ['-cfs']
|
17
|
+
t.rcov = true
|
18
|
+
t.rcov_opts = ['--exclude', 'gems,spec/,examples/']
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'spec/rake/verify_rcov'
|
22
|
+
RCov::VerifyTask.new(:verify_rcov => 'spec:rcov') do |t|
|
23
|
+
t.threshold = 77.0
|
24
|
+
t.require_exact_threshold = false
|
25
|
+
end
|
26
|
+
|
27
|
+
rescue LoadError
|
28
|
+
puts "spec targets require RSpec"
|
11
29
|
end
|
data/lib/sinmetrics.rb
CHANGED
@@ -0,0 +1,274 @@
|
|
1
|
+
#This class is outside code's main interface into the ABingo A/B testing framework.
|
2
|
+
#Unless you're fiddling with implementation details, it is the only one you need worry about.
|
3
|
+
|
4
|
+
#Usage of ABingo, including practical hints, is covered at http://www.bingocardcreator.com/abingo
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'sinatra/base'
|
8
|
+
require 'active_support'
|
9
|
+
rescue LoadError
|
10
|
+
retry if require 'rubygems'
|
11
|
+
raise
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
module Sinatra
|
16
|
+
class AbingoObject
|
17
|
+
|
18
|
+
#Defined options:
|
19
|
+
# :enable_specification => if true, allow params[test_name] to override the calculated value for a test.
|
20
|
+
def initialize app={}
|
21
|
+
if app.respond_to?(:options)
|
22
|
+
@app = app
|
23
|
+
[:identity, :enable_specification, :cache, :salt].each do |var|
|
24
|
+
begin
|
25
|
+
instance_variable_set("@#{var}", app.options.send("abingo_#{var}"))
|
26
|
+
rescue NoMethodError
|
27
|
+
end
|
28
|
+
end
|
29
|
+
else
|
30
|
+
[:identity, :enable_specification, :cache, :salt].each do |var|
|
31
|
+
instance_variable_set("@#{var}", app[var]) if app.has_key?(var)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
@identity ||= rand(10 ** 10).to_i.to_s
|
35
|
+
@cache ||= ActiveSupport::Cache::MemoryStore.new
|
36
|
+
@salt ||= ''
|
37
|
+
end
|
38
|
+
|
39
|
+
attr_reader :app, :enable_specification, :cache, :salt
|
40
|
+
attr_accessor :identity
|
41
|
+
|
42
|
+
#A simple convenience method for doing an A/B test. Returns true or false.
|
43
|
+
#If you pass it a block, it will bind the choice to the variable given to the block.
|
44
|
+
def flip(test_name)
|
45
|
+
if block_given?
|
46
|
+
yield(test(test_name, [true, false]))
|
47
|
+
else
|
48
|
+
test(test_name, [true, false])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
#This is the meat of A/Bingo.
|
53
|
+
#options accepts
|
54
|
+
# :multiple_participation (true or false)
|
55
|
+
# :conversion name of conversion to listen for (alias: conversion_name)
|
56
|
+
def test(test_name, alternatives, options = {})
|
57
|
+
short_circuit = cache.read("Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_"))
|
58
|
+
unless short_circuit.nil?
|
59
|
+
return short_circuit #Test has been stopped, pick canonical alternative.
|
60
|
+
end
|
61
|
+
|
62
|
+
unless Experiment.exists?(self, test_name)
|
63
|
+
conversion_name = options[:conversion] || options[:conversion_name]
|
64
|
+
Experiment.start_experiment!(self, test_name, parse_alternatives(alternatives), conversion_name)
|
65
|
+
end
|
66
|
+
|
67
|
+
choice = find_alternative_for_user(test_name, alternatives)
|
68
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
69
|
+
|
70
|
+
#Set this user to participate in this experiment, and increment participants count.
|
71
|
+
if options[:multiple_participation] || !(participating_tests.include?(test_name))
|
72
|
+
participating_tests = participating_tests.dup if participating_tests.frozen?
|
73
|
+
unless participating_tests.include?(test_name)
|
74
|
+
participating_tests << test_name
|
75
|
+
cache.write("Abingo::participating_tests::#{identity}", participating_tests)
|
76
|
+
end
|
77
|
+
Alternative.score_participation(self, test_name)
|
78
|
+
end
|
79
|
+
|
80
|
+
if block_given?
|
81
|
+
yield(choice)
|
82
|
+
else
|
83
|
+
choice
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
#Scores conversions for tests.
|
88
|
+
#test_name_or_array supports three types of input:
|
89
|
+
#
|
90
|
+
#A conversion name: scores a conversion for any test the user is participating in which
|
91
|
+
# is listening to the specified conversion.
|
92
|
+
#
|
93
|
+
#A test name: scores a conversion for the named test if the user is participating in it.
|
94
|
+
#
|
95
|
+
#An array of either of the above: for each element of the array, process as above.
|
96
|
+
#
|
97
|
+
#nil: score a conversion for every test the u
|
98
|
+
def bingo!(name = nil, options = {})
|
99
|
+
if name.kind_of? Array
|
100
|
+
name.map do |single_test|
|
101
|
+
self.bingo!(single_test, options)
|
102
|
+
end
|
103
|
+
else
|
104
|
+
if name.nil?
|
105
|
+
#Score all participating tests
|
106
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
107
|
+
participating_tests.each do |participating_test|
|
108
|
+
self.bingo!(participating_test, options)
|
109
|
+
end
|
110
|
+
else #Could be a test name or conversion name.
|
111
|
+
conversion_name = name.gsub(" ", "_")
|
112
|
+
tests_listening_to_conversion = cache.read("Abingo::tests_listening_to_conversion#{conversion_name}")
|
113
|
+
if tests_listening_to_conversion
|
114
|
+
if tests_listening_to_conversion.size > 1
|
115
|
+
tests_listening_to_conversion.map do |individual_test|
|
116
|
+
self.score_conversion!(individual_test.to_s, options)
|
117
|
+
end
|
118
|
+
elsif tests_listening_to_conversion.size == 1
|
119
|
+
test_name_str = tests_listening_to_conversion.first.to_s
|
120
|
+
self.score_conversion!(test_name_str, options)
|
121
|
+
end
|
122
|
+
else
|
123
|
+
#No tests listening for this conversion. Assume it is just a test name.
|
124
|
+
test_name_str = name.to_s
|
125
|
+
self.score_conversion!(test_name_str, options)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
#protected
|
132
|
+
|
133
|
+
#For programmer convenience, we allow you to specify what the alternatives for
|
134
|
+
#an experiment are in a few ways. Thus, we need to actually be able to handle
|
135
|
+
#all of them. We fire this parser very infrequently (once per test, typically)
|
136
|
+
#so it can be as complicated as we want.
|
137
|
+
# Integer => a number 1 through N
|
138
|
+
# Range => a number within the range
|
139
|
+
# Array => an element of the array.
|
140
|
+
# Hash => assumes a hash of something to int. We pick one of the
|
141
|
+
# somethings, weighted accorded to the ints provided. e.g.
|
142
|
+
# {:a => 2, :b => 3} produces :a 40% of the time, :b 60%.
|
143
|
+
#
|
144
|
+
#Alternatives are always represented internally as an array.
|
145
|
+
def parse_alternatives(alternatives)
|
146
|
+
if alternatives.kind_of? Array
|
147
|
+
return alternatives
|
148
|
+
elsif alternatives.kind_of? Integer
|
149
|
+
return (1..alternatives).to_a
|
150
|
+
elsif alternatives.kind_of? Range
|
151
|
+
return alternatives.to_a
|
152
|
+
elsif alternatives.kind_of? Hash
|
153
|
+
alternatives_array = []
|
154
|
+
alternatives.each do |key, value|
|
155
|
+
if value.kind_of? Integer
|
156
|
+
alternatives_array += [key] * value
|
157
|
+
else
|
158
|
+
raise "You gave a hash with #{key} => #{value} as an element. The value must be an integral weight."
|
159
|
+
end
|
160
|
+
end
|
161
|
+
return alternatives_array
|
162
|
+
else
|
163
|
+
raise "I don't know how to turn [#{alternatives}] into an array of alternatives."
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def retrieve_alternatives(test_name, alternatives)
|
168
|
+
cache_key = "Abingo::Experiment::#{test_name}::alternatives".gsub(" ","_")
|
169
|
+
alternative_array = self.cache.fetch(cache_key) do
|
170
|
+
self.parse_alternatives(alternatives)
|
171
|
+
end
|
172
|
+
alternative_array
|
173
|
+
end
|
174
|
+
|
175
|
+
def find_alternative_for_user(test_name, alternatives)
|
176
|
+
alternatives_array = retrieve_alternatives(test_name, alternatives)
|
177
|
+
alternatives_array[self.modulo_choice(test_name, alternatives_array.size)]
|
178
|
+
end
|
179
|
+
|
180
|
+
#Quickly determines what alternative to show a given user. Given a test name
|
181
|
+
#and their identity, we hash them together (which, for MD5, provably introduces
|
182
|
+
#enough entropy that we don't care) otherwise
|
183
|
+
def modulo_choice(test_name, choices_count)
|
184
|
+
Digest::MD5.hexdigest(salt.to_s + test_name + self.identity.to_s).to_i(16) % choices_count
|
185
|
+
end
|
186
|
+
|
187
|
+
def score_conversion!(test_name, options = {})
|
188
|
+
test_name.gsub!(" ", "_")
|
189
|
+
participating_tests = cache.read("Abingo::participating_tests::#{identity}") || []
|
190
|
+
if options[:assume_participation] || participating_tests.include?(test_name)
|
191
|
+
cache_key = "Abingo::conversions(#{identity},#{test_name}"
|
192
|
+
if options[:multiple_conversions] || !cache.read(cache_key)
|
193
|
+
Alternative.score_conversion(self, test_name)
|
194
|
+
if cache.exist?(cache_key)
|
195
|
+
cache.increment(cache_key)
|
196
|
+
else
|
197
|
+
cache.write(cache_key, 1)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
module AbingoObject::ConversionRate
|
205
|
+
def conversion_rate
|
206
|
+
1.0 * conversions / participants
|
207
|
+
end
|
208
|
+
|
209
|
+
def pretty_conversion_rate
|
210
|
+
sprintf("%4.2f%%", conversion_rate * 100)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
module AbingoHelper
|
215
|
+
def abingo
|
216
|
+
env['abingo.helper'] ||= AbingoObject.new(self)
|
217
|
+
end
|
218
|
+
|
219
|
+
def ab_test(test_name, alternatives = nil, options = {})
|
220
|
+
if (abingo.enable_specification && !params[test_name].blank?)
|
221
|
+
choice = params[test_name]
|
222
|
+
elsif (alternatives.nil?)
|
223
|
+
choice = abingo.flip(test_name)
|
224
|
+
else
|
225
|
+
choice = abingo.test(test_name, alternatives, options)
|
226
|
+
end
|
227
|
+
|
228
|
+
if block_given?
|
229
|
+
yield(choice)
|
230
|
+
else
|
231
|
+
choice
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
alias ab abingo
|
236
|
+
end
|
237
|
+
|
238
|
+
class AbingoSettings
|
239
|
+
def initialize app, &blk
|
240
|
+
@app = app
|
241
|
+
instance_eval &blk
|
242
|
+
end
|
243
|
+
%w[identity backend enable_specification cache salt].each do |param|
|
244
|
+
class_eval %[
|
245
|
+
def #{param} val, &blk
|
246
|
+
@app.set :abingo_#{param}, val
|
247
|
+
end
|
248
|
+
]
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
module Abingo
|
253
|
+
def abingo &blk
|
254
|
+
AbingoSettings.new(self, &blk)
|
255
|
+
end
|
256
|
+
|
257
|
+
def self.registered app
|
258
|
+
app.helpers AbingoHelper
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
Application.register Abingo
|
263
|
+
end
|
264
|
+
|
265
|
+
Abingo = Sinatra::AbingoObject
|
266
|
+
|
267
|
+
require 'dm-core'
|
268
|
+
require 'dm-aggregates'
|
269
|
+
require 'dm-observer'
|
270
|
+
require 'dm-timestamps'
|
271
|
+
require 'dm-adjust'
|
272
|
+
require 'abingo/statistics'
|
273
|
+
require 'abingo/alternative'
|
274
|
+
require 'abingo/experiment'
|
@@ -0,0 +1,30 @@
|
|
1
|
+
class Abingo::Alternative
|
2
|
+
include Abingo::ConversionRate
|
3
|
+
include DataMapper::Resource
|
4
|
+
|
5
|
+
property :id, Serial
|
6
|
+
property :content, String
|
7
|
+
property :lookup, String, :length => 32, :index => true
|
8
|
+
property :weight, Integer, :default => 1
|
9
|
+
property :participants, Integer, :default => 0
|
10
|
+
property :conversions, Integer, :default => 0
|
11
|
+
|
12
|
+
belongs_to :experiment
|
13
|
+
|
14
|
+
def self.calculate_lookup(abingo, test_name, alternative_name)
|
15
|
+
Digest::MD5.hexdigest(abingo.salt + test_name + alternative_name.to_s)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.score_conversion(abingo, test_name)
|
19
|
+
viewed_alternative = abingo.find_alternative_for_user(test_name,
|
20
|
+
Abingo::Experiment.alternatives_for_test(abingo, test_name))
|
21
|
+
all(:lookup => self.calculate_lookup(abingo, test_name, viewed_alternative)).adjust!(:conversions => 1)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.score_participation(abingo, test_name)
|
25
|
+
viewed_alternative = abingo.find_alternative_for_user(test_name,
|
26
|
+
Abingo::Experiment.alternatives_for_test(abingo, test_name))
|
27
|
+
all(:lookup => self.calculate_lookup(abingo, test_name, viewed_alternative)).adjust!(:participants => 1)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
class Abingo::Experiment
|
2
|
+
include DataMapper::Resource
|
3
|
+
include Abingo::Statistics
|
4
|
+
include Abingo::ConversionRate
|
5
|
+
|
6
|
+
property :test_name, String, :key => true
|
7
|
+
property :status, String
|
8
|
+
|
9
|
+
has n, :alternatives, "Alternative"
|
10
|
+
|
11
|
+
def cache_keys
|
12
|
+
["Abingo::Experiment::exists(#{test_name})".gsub(" ", "_"),
|
13
|
+
"Abingo::Experiment::#{test_name}::alternatives".gsub(" ","_"),
|
14
|
+
"Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_")
|
15
|
+
]
|
16
|
+
end
|
17
|
+
|
18
|
+
def before_destroy
|
19
|
+
cache_keys.each do |key|
|
20
|
+
Abingo.cache.delete key
|
21
|
+
end
|
22
|
+
true
|
23
|
+
end
|
24
|
+
|
25
|
+
def participants
|
26
|
+
alternatives.sum(:participants)
|
27
|
+
end
|
28
|
+
|
29
|
+
def conversions
|
30
|
+
alternatives.sum(:conversions)
|
31
|
+
end
|
32
|
+
|
33
|
+
def best_alternative
|
34
|
+
alternatives.max do |a,b|
|
35
|
+
a.conversion_rate <=> b.conversion_rate
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.exists?(abingo, test_name)
|
40
|
+
cache_key = "Abingo::Experiment::exists(#{test_name})".gsub(" ", "_")
|
41
|
+
ret = abingo.cache.fetch(cache_key) do
|
42
|
+
count = Abingo::Experiment.count(:conditions => {:test_name => test_name})
|
43
|
+
count > 0 ? count : nil
|
44
|
+
end
|
45
|
+
(!ret.nil?)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.alternatives_for_test(abingo, test_name)
|
49
|
+
cache_key = "Abingo::#{test_name}::alternatives".gsub(" ","_")
|
50
|
+
abingo.cache.fetch(cache_key) do
|
51
|
+
experiment = Abingo::Experiment.first(:test_name => test_name)
|
52
|
+
alternatives_array = abingo.cache.fetch(cache_key) do
|
53
|
+
tmp_array = experiment.alternatives.map do |alt|
|
54
|
+
[alt.content, alt.weight]
|
55
|
+
end
|
56
|
+
tmp_hash = tmp_array.inject({}) {|hash, couplet| hash[couplet[0]] = couplet[1]; hash}
|
57
|
+
abingo.parse_alternatives(tmp_hash)
|
58
|
+
end
|
59
|
+
alternatives_array
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.start_experiment!(abingo, test_name, alternatives_array, conversion_name = nil)
|
64
|
+
conversion_name ||= test_name
|
65
|
+
conversion_name.gsub!(" ", "_")
|
66
|
+
cloned_alternatives_array = alternatives_array.clone
|
67
|
+
Abingo::Experiment.transaction do |txn|
|
68
|
+
experiment = Abingo::Experiment.first_or_create(:test_name => test_name)
|
69
|
+
experiment.alternatives.destroy #Blows away alternatives for pre-existing experiments.
|
70
|
+
while (cloned_alternatives_array.size > 0)
|
71
|
+
alt = cloned_alternatives_array[0]
|
72
|
+
weight = cloned_alternatives_array.size - (cloned_alternatives_array - [alt]).size
|
73
|
+
experiment.alternatives << Abingo::Alternative.new(:content => alt, :weight => weight,
|
74
|
+
:lookup => Abingo::Alternative.calculate_lookup(abingo, test_name, alt))
|
75
|
+
cloned_alternatives_array -= [alt]
|
76
|
+
end
|
77
|
+
experiment.status = "Live"
|
78
|
+
experiment.save
|
79
|
+
abingo.cache.write("Abingo::Experiment::exists(#{test_name})".gsub(" ", "_"), 1)
|
80
|
+
|
81
|
+
#This might have issues in very, very high concurrency environments...
|
82
|
+
|
83
|
+
tests_listening_to_conversion = abingo.cache.read("Abingo::tests_listening_to_conversion#{conversion_name}") || []
|
84
|
+
tests_listening_to_conversion = tests_listening_to_conversion.dup if tests_listening_to_conversion.frozen?
|
85
|
+
tests_listening_to_conversion << test_name unless tests_listening_to_conversion.include? test_name
|
86
|
+
abingo.cache.write("Abingo::tests_listening_to_conversion#{conversion_name}", tests_listening_to_conversion)
|
87
|
+
experiment
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def end_experiment!(abingo, final_alternative, conversion_name = nil)
|
92
|
+
conversion_name ||= test_name
|
93
|
+
Abingo::Experiment.transaction do
|
94
|
+
alternatives.each do |alternative|
|
95
|
+
alternative.lookup = "Experiment completed. #{alternative.id}"
|
96
|
+
alternative.save!
|
97
|
+
end
|
98
|
+
update(:status => "Finished")
|
99
|
+
abingo.cache.write("Abingo::Experiment::short_circuit(#{test_name})".gsub(" ", "_"), final_alternative)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
#Designed to be included into Abingo::Experiment, but you can feel free to adapt this
|
2
|
+
#to anything you want.
|
3
|
+
|
4
|
+
module Abingo::Statistics
|
5
|
+
|
6
|
+
HANDY_Z_SCORE_CHEATSHEET = [[0.10, 1.29], [0.05, 1.65], [0.01, 2.33], [0.001, 3.08]]
|
7
|
+
|
8
|
+
PERCENTAGES = {0.10 => '90%', 0.05 => '95%', 0.01 => '99%', 0.001 => '99.9%'}
|
9
|
+
|
10
|
+
DESCRIPTION_IN_WORDS = {0.10 => 'fairly confident', 0.05 => 'confident',
|
11
|
+
0.01 => 'very confident', 0.001 => 'extremely confident'}
|
12
|
+
def zscore
|
13
|
+
if alternatives.size != 2
|
14
|
+
raise "Sorry, can't currently automatically calculate statistics for A/B tests with > 2 alternatives."
|
15
|
+
end
|
16
|
+
|
17
|
+
if (alternatives[0].participants == 0) || (alternatives[1].participants == 0)
|
18
|
+
raise "Can't calculate the z score if either of the alternatives lacks participants."
|
19
|
+
end
|
20
|
+
|
21
|
+
cr1 = alternatives[0].conversion_rate
|
22
|
+
cr2 = alternatives[1].conversion_rate
|
23
|
+
|
24
|
+
n1 = alternatives[0].participants
|
25
|
+
n2 = alternatives[1].participants
|
26
|
+
|
27
|
+
numerator = cr1 - cr2
|
28
|
+
frac1 = cr1 * (1 - cr1) / n1
|
29
|
+
frac2 = cr2 * (1 - cr2) / n2
|
30
|
+
|
31
|
+
numerator / ((frac1 + frac2) ** 0.5)
|
32
|
+
end
|
33
|
+
|
34
|
+
def p_value
|
35
|
+
index = 0
|
36
|
+
z = zscore
|
37
|
+
z = z.abs
|
38
|
+
found_p = nil
|
39
|
+
while index < HANDY_Z_SCORE_CHEATSHEET.size do
|
40
|
+
if (z > HANDY_Z_SCORE_CHEATSHEET[index][1])
|
41
|
+
found_p = HANDY_Z_SCORE_CHEATSHEET[index][0]
|
42
|
+
end
|
43
|
+
index += 1
|
44
|
+
end
|
45
|
+
found_p
|
46
|
+
end
|
47
|
+
|
48
|
+
def is_statistically_significant?(p = 0.05)
|
49
|
+
p_value <= p
|
50
|
+
end
|
51
|
+
|
52
|
+
def pretty_conversion_rate
|
53
|
+
sprintf("%4.2f%%", conversion_rate * 100)
|
54
|
+
end
|
55
|
+
|
56
|
+
def describe_result_in_words
|
57
|
+
begin
|
58
|
+
z = zscore
|
59
|
+
rescue
|
60
|
+
return "Could not execute the significance test because one or more of the alternatives has not been seen yet."
|
61
|
+
end
|
62
|
+
p = p_value
|
63
|
+
|
64
|
+
words = ""
|
65
|
+
if (alternatives[0].participants < 10) || (alternatives[1].participants < 10)
|
66
|
+
words += "Take these results with a grain of salt since your samples are so small: "
|
67
|
+
end
|
68
|
+
|
69
|
+
alts = alternatives - [best_alternative]
|
70
|
+
worst_alternative = alts.first
|
71
|
+
|
72
|
+
words += "The best alternative you have is: [#{best_alternative.content}], which had "
|
73
|
+
words += "#{best_alternative.conversions} conversions from #{best_alternative.participants} participants "
|
74
|
+
words += "(#{best_alternative.pretty_conversion_rate}). The other alternative was [#{worst_alternative.content}], "
|
75
|
+
words += "which had #{worst_alternative.conversions} conversions from #{worst_alternative.participants} participants "
|
76
|
+
words += "(#{worst_alternative.pretty_conversion_rate}). "
|
77
|
+
|
78
|
+
if (p.nil?)
|
79
|
+
words += "However, this difference is not statistically significant."
|
80
|
+
else
|
81
|
+
words += "This difference is #{PERCENTAGES[p]} likely to be statistically significant, which means you can be "
|
82
|
+
words += "#{DESCRIPTION_IN_WORDS[p]} that it is the result of your alternatives actually mattering, rather than "
|
83
|
+
words += "being due to random chance. However, this statistical test can't measure how likely the currently "
|
84
|
+
words += "observed magnitude of the difference is to be accurate or not. It only says \"better\", not \"better "
|
85
|
+
words += "by so much\"."
|
86
|
+
end
|
87
|
+
words
|
88
|
+
end
|
89
|
+
|
90
|
+
end
|
data/spec/abingo_spec.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Abingo' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
ab = Abingo.new :identity => 'abcd'
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class AbingoApp < Sinatra::Base
|
11
|
+
register Sinatra::Abingo
|
12
|
+
abingo do
|
13
|
+
identity 'abcd'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
app = AbingoApp.new
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'Abingo Specs' do
|
23
|
+
before :each do
|
24
|
+
@abingo = Abingo.new
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should automatically assign identity" do
|
28
|
+
@abingo.identity.should_not be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should parse alternatives" do
|
32
|
+
array = %w{a b c}
|
33
|
+
@abingo.parse_alternatives(array).should == array
|
34
|
+
@abingo.parse_alternatives(65).size.should == 65
|
35
|
+
@abingo.parse_alternatives(2..5).size.should == 4
|
36
|
+
@abingo.parse_alternatives(2..5).should_not include(1)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should create experiments" do
|
40
|
+
Abingo::Experiment.count.should == 0
|
41
|
+
Abingo::Alternative.count.should == 0
|
42
|
+
alternatives = %w{A B}
|
43
|
+
alternative_selected = @abingo.test("unit_test_sample_A", alternatives)
|
44
|
+
Abingo::Experiment.count.should == 1
|
45
|
+
Abingo::Alternative.count.should == 2
|
46
|
+
alternatives.should include(alternative_selected)
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should be able to call exists" do
|
50
|
+
@abingo.test("exist words right", %w{does does_not})
|
51
|
+
Abingo::Experiment.exists?(@abingo, "exist words right").should be_true
|
52
|
+
Abingo::Experiment.exists?(@abingo, "other words right").should_not be_true
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should pick alternatives consistently" do
|
56
|
+
alternative_picked = @abingo.test("consistency_test", 1..100)
|
57
|
+
100.times do
|
58
|
+
@abingo.test("consistency_test", 1..100).should == alternative_picked
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should have working participation" do
|
63
|
+
new_tests = %w{participationA participationB participationC}
|
64
|
+
new_tests.map do |test_name|
|
65
|
+
@abingo.test(test_name, 1..5)
|
66
|
+
end
|
67
|
+
|
68
|
+
participating_tests = @abingo.cache.read("Abingo::participating_tests::#{@abingo.identity}") || []
|
69
|
+
|
70
|
+
new_tests.map do |test_name|
|
71
|
+
participating_tests.should include(test_name)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should count participants" do
|
76
|
+
test_name = "participants_counted_test"
|
77
|
+
alternative = @abingo.test(test_name, %w{a b c})
|
78
|
+
|
79
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
80
|
+
lookup = Abingo::Alternative.calculate_lookup(@abingo, test_name, alternative)
|
81
|
+
chosen_alt = Abingo::Alternative.first(:lookup => lookup)
|
82
|
+
ex.participants.should == 1
|
83
|
+
chosen_alt.participants.should == 1
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should track conversions by test name" do
|
87
|
+
test_name = "conversion_test_by_name"
|
88
|
+
alternative = @abingo.test(test_name, %w{a b c})
|
89
|
+
@abingo.bingo!(test_name)
|
90
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
91
|
+
lookup = Abingo::Alternative.calculate_lookup(@abingo, test_name, alternative)
|
92
|
+
chosen_alt = Abingo::Alternative.first(:lookup => lookup)
|
93
|
+
ex.conversions.should == 1
|
94
|
+
chosen_alt.conversions.should == 1
|
95
|
+
|
96
|
+
@abingo.bingo!(test_name)
|
97
|
+
|
98
|
+
#Should still only have one because this conversion should not be double counted.
|
99
|
+
#We haven't specified that in the test options.
|
100
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
101
|
+
ex.conversions.should == 1
|
102
|
+
end
|
103
|
+
|
104
|
+
it "should track conversions by conversion name" do
|
105
|
+
conversion_name = "purchase"
|
106
|
+
tests = %w{conversionTrackingByConversionNameA conversionTrackingByConversionNameB conversionTrackingByConversionNameC}
|
107
|
+
tests.map do |test_name|
|
108
|
+
@abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
109
|
+
end
|
110
|
+
|
111
|
+
@abingo.bingo!(conversion_name)
|
112
|
+
tests.map do |test_name|
|
113
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
114
|
+
ex.conversions.should == 1
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should be possible to short circuit tests" do
|
119
|
+
conversion_name = "purchase"
|
120
|
+
test_name = "short circuit test"
|
121
|
+
alt_picked = @abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
122
|
+
ex = Abingo::Experiment.first(:test_name => test_name)
|
123
|
+
alt_not_picked = (%w{A B} - [alt_picked]).first
|
124
|
+
|
125
|
+
ex.end_experiment!(@abingo, alt_not_picked, conversion_name)
|
126
|
+
|
127
|
+
ex.reload
|
128
|
+
ex.status.should == "Finished"
|
129
|
+
|
130
|
+
@abingo.bingo!(test_name) #Should not be counted, test is over.
|
131
|
+
ex.conversions.should == 0
|
132
|
+
|
133
|
+
old_identity = @abingo.identity
|
134
|
+
@abingo.identity = "shortCircuitTestNewIdentity"
|
135
|
+
@abingo.test(test_name, %w{A B}, :conversion => conversion_name)
|
136
|
+
@abingo.identity = old_identity
|
137
|
+
ex.reload
|
138
|
+
|
139
|
+
# Original identity counted, new identity not counted b/c test stopped
|
140
|
+
ex.participants.should == 1
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Kontagent' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
kontagent = Kontagent.new :api_key => '1234', :secret => '5678'
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class KontagentApp < Sinatra::Base
|
11
|
+
register Sinatra::Kontagent
|
12
|
+
kontagent do
|
13
|
+
api_key "1234"
|
14
|
+
secret "5678"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
app = KontagentApp.new
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
end
|
23
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Sinatra Mixpanel' do
|
4
|
+
|
5
|
+
it 'should be configurable standalone' do
|
6
|
+
mp = Mixpanel.new :token => "abcdefghijklmnopqrstuvwyxz"
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should be configurable w/in sinatra' do
|
10
|
+
class MixpanelApp < Sinatra::Base
|
11
|
+
register Sinatra::Mixpanel
|
12
|
+
mixpanel do
|
13
|
+
token "abcdefghijklmnopqrstuvwyxz"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
app = MixpanelApp.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'spec'
|
5
|
+
require 'sinmetrics'
|
6
|
+
|
7
|
+
# establish in-memory database for testing
|
8
|
+
DataMapper.setup(:default, "sqlite3::memory:")
|
9
|
+
|
10
|
+
Spec::Runner.configure do |config|
|
11
|
+
# reset database before each example is run
|
12
|
+
config.before(:each) { DataMapper.auto_migrate! }
|
13
|
+
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 5
|
9
|
+
version: 0.0.5
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Luke Petre
|
@@ -14,11 +14,94 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-04-
|
17
|
+
date: 2010-04-14 00:00:00 +01:00
|
18
18
|
default_executable:
|
19
|
-
dependencies:
|
20
|
-
|
21
|
-
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: dm-core
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
- 10
|
30
|
+
- 2
|
31
|
+
version: 0.10.2
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: dm-aggregates
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 0
|
43
|
+
- 10
|
44
|
+
- 2
|
45
|
+
version: 0.10.2
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: dm-observer
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 0
|
57
|
+
- 10
|
58
|
+
- 2
|
59
|
+
version: 0.10.2
|
60
|
+
type: :runtime
|
61
|
+
version_requirements: *id003
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: dm-timestamps
|
64
|
+
prerelease: false
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
segments:
|
70
|
+
- 0
|
71
|
+
- 10
|
72
|
+
- 2
|
73
|
+
version: 0.10.2
|
74
|
+
type: :runtime
|
75
|
+
version_requirements: *id004
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: dm-adjust
|
78
|
+
prerelease: false
|
79
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
segments:
|
84
|
+
- 0
|
85
|
+
- 10
|
86
|
+
- 2
|
87
|
+
version: 0.10.2
|
88
|
+
type: :runtime
|
89
|
+
version_requirements: *id005
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: activesupport
|
92
|
+
prerelease: false
|
93
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ~>
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
segments:
|
98
|
+
- 2
|
99
|
+
- 3
|
100
|
+
- 5
|
101
|
+
version: 2.3.5
|
102
|
+
type: :runtime
|
103
|
+
version_requirements: *id006
|
104
|
+
description: A full-featured metrics extension for the sinatra webapp framework
|
22
105
|
email: lpetre@gmail.com
|
23
106
|
executables: []
|
24
107
|
|
@@ -27,25 +110,27 @@ extensions: []
|
|
27
110
|
extra_rdoc_files: []
|
28
111
|
|
29
112
|
files:
|
30
|
-
-
|
113
|
+
- .gitignore
|
31
114
|
- README
|
32
115
|
- Rakefile
|
33
116
|
- lib/sinmetrics.rb
|
117
|
+
- lib/sinmetrics/abingo.rb
|
118
|
+
- lib/sinmetrics/abingo/alternative.rb
|
119
|
+
- lib/sinmetrics/abingo/experiment.rb
|
120
|
+
- lib/sinmetrics/abingo/statistics.rb
|
34
121
|
- lib/sinmetrics/kontagent.rb
|
35
122
|
- lib/sinmetrics/mixpanel.rb
|
36
|
-
-
|
123
|
+
- spec/abingo_spec.rb
|
124
|
+
- spec/kontagent_spec.rb
|
125
|
+
- spec/mixpanel_spec.rb
|
126
|
+
- spec/spec_helper.rb
|
37
127
|
has_rdoc: true
|
38
128
|
homepage: http://github.com/lpetre/sinmetrics
|
39
129
|
licenses: []
|
40
130
|
|
41
131
|
post_install_message:
|
42
|
-
rdoc_options:
|
43
|
-
|
44
|
-
- --inline-source
|
45
|
-
- --title
|
46
|
-
- Sinmetrics
|
47
|
-
- --main
|
48
|
-
- README
|
132
|
+
rdoc_options: []
|
133
|
+
|
49
134
|
require_paths:
|
50
135
|
- lib
|
51
136
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -60,15 +145,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
60
145
|
- - ">="
|
61
146
|
- !ruby/object:Gem::Version
|
62
147
|
segments:
|
63
|
-
-
|
64
|
-
|
65
|
-
version: "1.2"
|
148
|
+
- 0
|
149
|
+
version: "0"
|
66
150
|
requirements: []
|
67
151
|
|
68
|
-
rubyforge_project:
|
152
|
+
rubyforge_project:
|
69
153
|
rubygems_version: 1.3.6
|
70
154
|
signing_key:
|
71
155
|
specification_version: 3
|
72
|
-
summary:
|
156
|
+
summary: simple sinatra metrics extension
|
73
157
|
test_files: []
|
74
158
|
|
data/Manifest
DELETED
data/sinmetrics.gemspec
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
# -*- encoding: utf-8 -*-
|
2
|
-
|
3
|
-
Gem::Specification.new do |s|
|
4
|
-
s.name = %q{sinmetrics}
|
5
|
-
s.version = "0.0.4"
|
6
|
-
|
7
|
-
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
|
-
s.authors = ["Luke Petre"]
|
9
|
-
s.date = %q{2010-04-05}
|
10
|
-
s.description = %q{Some metrics helpers for the Sinatra web framework}
|
11
|
-
s.email = %q{lpetre@gmail.com}
|
12
|
-
s.files = ["Manifest", "README", "Rakefile", "lib/sinmetrics.rb", "lib/sinmetrics/kontagent.rb", "lib/sinmetrics/mixpanel.rb", "sinmetrics.gemspec"]
|
13
|
-
s.homepage = %q{http://github.com/lpetre/sinmetrics}
|
14
|
-
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Sinmetrics", "--main", "README"]
|
15
|
-
s.require_paths = ["lib"]
|
16
|
-
s.rubyforge_project = %q{sinmetrics}
|
17
|
-
s.rubygems_version = %q{1.3.6}
|
18
|
-
s.summary = %q{Some metrics helpers for the Sinatra web framework}
|
19
|
-
|
20
|
-
if s.respond_to? :specification_version then
|
21
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
22
|
-
s.specification_version = 3
|
23
|
-
|
24
|
-
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
25
|
-
else
|
26
|
-
end
|
27
|
-
else
|
28
|
-
end
|
29
|
-
end
|