conductor 0.0.0 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +2 -4
- data/VERSION +1 -1
- data/generators/conductor_migration/conductor_migration_generator.rb +10 -0
- data/generators/conductor_migration/templates/conductor_migration.rb +56 -0
- data/init.rb +4 -0
- data/lib/conductor.rb +41 -0
- data/lib/conductor/experiment.rb +156 -0
- data/lib/conductor/models/daily_experiment.rb +17 -0
- data/lib/conductor/models/raw_experiment.rb +22 -0
- data/lib/conductor/models/weight_history.rb +14 -0
- data/lib/conductor/models/weighted_experiment.rb +15 -0
- data/lib/conductor/roll_up.rb +25 -0
- data/lib/conductor/weights.rb +91 -0
- data/test/test_conductor.rb +221 -3
- metadata +15 -8
- data/.document +0 -5
- data/.gitignore +0 -21
- data/conductor.gemspec +0 -52
data/Rakefile
CHANGED
@@ -5,18 +5,16 @@ begin
|
|
5
5
|
require 'jeweler'
|
6
6
|
Jeweler::Tasks.new do |gem|
|
7
7
|
gem.name = "conductor"
|
8
|
-
gem.summary = %Q{
|
8
|
+
gem.summary = %Q{lets you just try things while always maximizing towards a goal (e.g. purchase, signups, etc)}
|
9
9
|
gem.description = %Q{Conductor is the bastard child of a/b testing and personalization. It throws everything you know about creating a web site our the window and lets you just "try stuff" without ever having to worry about not maximing your site's "purpose." Have a new landing page? Just throw it to the conductor. Want to try different price points - conductor. Different form designs? Conductor. Conductor will rotate all alternatives through the mix and eventually settle on the top performing of all, without you having to do anything other than just creating. Think "intelligent A/B testing" on steriods.}
|
10
10
|
gem.email = "jlippiner@noctivity.com"
|
11
11
|
gem.homepage = "http://github.com/noctivityinc/conductor"
|
12
12
|
gem.authors = ["Noctivity"]
|
13
13
|
gem.rubyforge_project = "conductor"
|
14
|
+
gem.files = FileList["[A-Z]*", "{generators,lib}/**/*", "init.rb"]
|
14
15
|
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
15
16
|
end
|
16
17
|
Jeweler::GemcutterTasks.new
|
17
|
-
Jeweler::RubyforgeTasks.new do |rubyforge|
|
18
|
-
rubyforge.doc_task = "rdoc"
|
19
|
-
end
|
20
18
|
rescue LoadError
|
21
19
|
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
22
20
|
end
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.9
|
@@ -0,0 +1,10 @@
|
|
1
|
+
class ConductorMigrationGenerator < Rails::Generator::Base
|
2
|
+
require 'conductor'
|
3
|
+
|
4
|
+
def manifest
|
5
|
+
record do |m|
|
6
|
+
m.migration_template 'conductor_migration.rb', 'db/migrate',
|
7
|
+
:assigns => {:version => Conductor.MAJOR_VERSION.gsub(".", "")}
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
#Creates the database tables, plus indexes, you'll need to use Conductor.
|
2
|
+
|
3
|
+
class ConductorMigration<%= version -%>< ActiveRecord::Migration
|
4
|
+
def self.up
|
5
|
+
|
6
|
+
create_table "conductor_daily_experiments", :force => true do |t|
|
7
|
+
t.date "activity_date"
|
8
|
+
t.string "group_name"
|
9
|
+
t.string "option_name"
|
10
|
+
t.decimal "conversion_value", :precision => 8, :scale => 2
|
11
|
+
t.integer "views"
|
12
|
+
t.integer "conversions"
|
13
|
+
end
|
14
|
+
|
15
|
+
add_index "conductor_daily_experiments", ["activity_date"], :name => "index_conductor_daily_experiments_on_activity_date"
|
16
|
+
add_index "conductor_daily_experiments", ["group_name"], :name => "index_conductor_daily_experiments_on_group_name"
|
17
|
+
|
18
|
+
create_table "conductor_raw_experiments", :force => true do |t|
|
19
|
+
t.string "identity_id"
|
20
|
+
t.string "group_name"
|
21
|
+
t.string "option_name"
|
22
|
+
t.decimal "conversion_value", :precision => 8, :scale => 2
|
23
|
+
t.datetime "created_at"
|
24
|
+
t.datetime "updated_at"
|
25
|
+
t.string "goal"
|
26
|
+
end
|
27
|
+
|
28
|
+
create_table "conductor_weight_histories", :force => true do |t|
|
29
|
+
t.string "group_name"
|
30
|
+
t.string "option_name"
|
31
|
+
t.decimal "weight", :precision => 8, :scale => 2
|
32
|
+
t.datetime "computed_at"
|
33
|
+
t.integer "launch_window"
|
34
|
+
end
|
35
|
+
|
36
|
+
add_index "conductor_weight_histories", ["computed_at", "group_name"], :name => "conductor_wh_date_and_group_ndx"
|
37
|
+
|
38
|
+
create_table "conductor_weighted_experiments", :force => true do |t|
|
39
|
+
t.string "group_name"
|
40
|
+
t.string "option_name"
|
41
|
+
t.decimal "weight", :precision => 8, :scale => 2
|
42
|
+
t.datetime "created_at"
|
43
|
+
t.datetime "updated_at"
|
44
|
+
end
|
45
|
+
|
46
|
+
add_index "conductor_weighted_experiments", ["group_name"], :name => "index_conductor_weighted_experiments_on_group_name"
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.down
|
51
|
+
drop_table :conductor_raw_experiments
|
52
|
+
drop_table :conductor_daily_experiments
|
53
|
+
drop_table :conductor_weight_histories
|
54
|
+
drop_table :conductor_weighted_experiments
|
55
|
+
end
|
56
|
+
end
|
data/init.rb
ADDED
data/lib/conductor.rb
CHANGED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'conductor/experiment'
|
2
|
+
require 'conductor/roll_up'
|
3
|
+
require 'conductor/weights'
|
4
|
+
|
5
|
+
class Conductor
|
6
|
+
MAX_WEIGHTING_FACTOR = 1.25
|
7
|
+
MINIMUM_LAUNCH_DAYS = 7
|
8
|
+
DBG = false
|
9
|
+
|
10
|
+
@@VERSION = "0.1.0"
|
11
|
+
@@MAJOR_VERSION = "1.0"
|
12
|
+
cattr_reader :VERSION
|
13
|
+
cattr_reader :MAJOR_VERSION
|
14
|
+
|
15
|
+
cattr_writer :cache
|
16
|
+
|
17
|
+
def self.cache
|
18
|
+
@@cache || Rails.cache
|
19
|
+
end
|
20
|
+
|
21
|
+
class << self
|
22
|
+
def identity=(value)
|
23
|
+
@conductor_identity = value
|
24
|
+
end
|
25
|
+
|
26
|
+
def identity
|
27
|
+
return (@conductor_identity || ActiveSupport::SecureRandom.hex(16))
|
28
|
+
end
|
29
|
+
|
30
|
+
def log(msg)
|
31
|
+
puts msg if DBG
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
class Array
|
38
|
+
def sum_it(attribute)
|
39
|
+
self.map {|x| x.send(attribute) }.compact.sum
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
class Conductor
|
2
|
+
class Experiment
|
3
|
+
class << self
|
4
|
+
# Selects the best option for a given group
|
5
|
+
#
|
6
|
+
# Method also saves the selection to the
|
7
|
+
# database so everything happens in one move
|
8
|
+
#
|
9
|
+
# options allow you to specify a specific GOAL
|
10
|
+
# in case you have multiple goals per site. For example,
|
11
|
+
# if you are using Conductor to maximize newsletter signups
|
12
|
+
# and orders, you might have:
|
13
|
+
# {:goal => "signup"}
|
14
|
+
# and
|
15
|
+
# {:goal => "purchase" }
|
16
|
+
#
|
17
|
+
# goals are important since you can specify which goal converted
|
18
|
+
# with the track! method to only update records for that
|
19
|
+
# specific goal.
|
20
|
+
#
|
21
|
+
def pick(group_name, alternatives, options={})
|
22
|
+
group_name = sanitize(group_name) # => clean up and standardize
|
23
|
+
|
24
|
+
# check for previous selection
|
25
|
+
selection = Conductor.cache.read("Conductor::#{Conductor.identity}::Experience::#{group_name}")
|
26
|
+
|
27
|
+
unless selection
|
28
|
+
selection = select_option_for_group(group_name, alternatives)
|
29
|
+
Conductor::RawExperiment.create!({:identity_id => Conductor.identity.to_s, :group_name => group_name, :option_name => selection}.merge!(options))
|
30
|
+
Conductor.cache.write("Conductor::#{Conductor.identity}::Experience::#{group_name}", selection)
|
31
|
+
end
|
32
|
+
|
33
|
+
return selection
|
34
|
+
end
|
35
|
+
|
36
|
+
# returns the raw weighting table for all alternatives for a specified group
|
37
|
+
def weights(group_name, alternatives)
|
38
|
+
group_name = sanitize(group_name) # => clean up and standardize
|
39
|
+
return generate_weighting_table(group_name, alternatives)
|
40
|
+
end
|
41
|
+
|
42
|
+
# Records a conversion for the visitor.
|
43
|
+
#
|
44
|
+
# May optionally supply a conversion value and goal.
|
45
|
+
#
|
46
|
+
# Conversion value is specified as {:value => 123.45}
|
47
|
+
#
|
48
|
+
# If no goal is specified as an option, than ALL selected alternatives
|
49
|
+
# for the experiments for a specific user will be updated with the
|
50
|
+
# conversion value. If a goal is specified, then only those
|
51
|
+
# records will be updated.
|
52
|
+
#
|
53
|
+
# To clarify by explaination -
|
54
|
+
#
|
55
|
+
# Assume you are selecting a landing page to maximize newsletter
|
56
|
+
# signups and are selecting a price point to maximize purchases.
|
57
|
+
# In this case you would have two goals -
|
58
|
+
# - signup
|
59
|
+
# - purchase
|
60
|
+
#
|
61
|
+
# Now, if we assume that for visitor 24601 a landing page and
|
62
|
+
# price point are selected before they signup for the newsletter,
|
63
|
+
# if you called track! after a newsletter signup occurred without
|
64
|
+
# specifying the goal, then a conversion would ALSO (and incorrectly)
|
65
|
+
# be recorded by the PURCHASE goal as well as the SIGNUP goal.
|
66
|
+
#
|
67
|
+
# What you needed to do was call track!({:goal => 'signup'}) to correctly record
|
68
|
+
# a conversion for visitor 24601 and the newsletter signup only.
|
69
|
+
#
|
70
|
+
def track!(options={})
|
71
|
+
value = (options.delete(:value) || 1) # => pull the conversion value and remove from hash or set value to 1
|
72
|
+
experiments = Conductor::RawExperiment.find(:all, :conditions => {:identity_id => Conductor.identity}.merge!(options))
|
73
|
+
experiments.each {|x| x.update_attributes(:conversion_value => value)} if experiments
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def select_option_for_group(group_name, alternatives)
|
79
|
+
# create weighting table
|
80
|
+
weighting_table = generate_weighting_table(group_name, alternatives)
|
81
|
+
|
82
|
+
# make a selection from weighted hash
|
83
|
+
return choose_weighted(weighting_table)
|
84
|
+
end
|
85
|
+
|
86
|
+
# Returns a hash of alternatives with weights based on conversion
|
87
|
+
# e.g. {:option_a => .25, :option_b => .25, :option_c => .50}
|
88
|
+
#
|
89
|
+
# Note: We create sql where statement that includes the list of
|
90
|
+
# alternatives to select from in case an existing group
|
91
|
+
# has an option you no longer want to include in the result set
|
92
|
+
#
|
93
|
+
# TODO: store all weights for a group in cache and then weed out
|
94
|
+
# those not in the alternatives list
|
95
|
+
#
|
96
|
+
def generate_weighting_table(group_name, alternatives)
|
97
|
+
# create the conditions after sanitizing sql.
|
98
|
+
option_conditions = alternatives.inject([]) {|res,x| res << "option_name = '#{sanitize(x)}'"}.join(' OR ')
|
99
|
+
|
100
|
+
conditions = "group_name = '#{group_name}' AND (#{option_conditions})"
|
101
|
+
|
102
|
+
# get the options from the database
|
103
|
+
weights ||= Conductor::WeightedExperiment.find(:all, :conditions => conditions)
|
104
|
+
|
105
|
+
# create selection hash
|
106
|
+
weighting_table = weights.inject({}) {|res, x| res.merge!({x.option_name => x.weight})}
|
107
|
+
|
108
|
+
# is anything missing?
|
109
|
+
options_names = weights.map(&:option_name)
|
110
|
+
missing = alternatives - options_names
|
111
|
+
|
112
|
+
# if anything is missing, add it to the weighted list
|
113
|
+
unless missing.empty?
|
114
|
+
max_weight = weights.empty? ? 1 : weights.max {|a,b| a.weight <=> b.weight}.weight
|
115
|
+
missing.each do |name|
|
116
|
+
weighting_table.merge!({name => max_weight * MAX_WEIGHTING_FACTOR})
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
return weighting_table
|
121
|
+
end
|
122
|
+
|
123
|
+
# selects a random float
|
124
|
+
def float_rand(start_num, end_num=0)
|
125
|
+
width = end_num-start_num
|
126
|
+
return (rand*width)+start_num
|
127
|
+
end
|
128
|
+
|
129
|
+
# choose a random option based on weights
|
130
|
+
# from recipe 5.11 in ruby cookbook
|
131
|
+
def choose_weighted(weighted)
|
132
|
+
sum = weighted.inject(0) do |sum, item_and_weight|
|
133
|
+
sum += item_and_weight[1]
|
134
|
+
end
|
135
|
+
target = float_rand(sum)
|
136
|
+
weighted.each do |item, weight|
|
137
|
+
return item if target <= weight
|
138
|
+
target -= weight
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def normalize!(weighted)
|
143
|
+
sum = weighted.inject(0) do |sum, item_and_weight|
|
144
|
+
sum += item_and_weight[1]
|
145
|
+
end
|
146
|
+
sum = sum.to_f
|
147
|
+
weighted.each { |item, weight| weighted[item] = weight/sum }
|
148
|
+
end
|
149
|
+
|
150
|
+
def sanitize(str)
|
151
|
+
str.gsub(/\s/,'_').downcase
|
152
|
+
end
|
153
|
+
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# == Schema Information
|
2
|
+
#
|
3
|
+
# Table name: conductor_daily_experiments
|
4
|
+
#
|
5
|
+
# id :integer not null, primary key
|
6
|
+
# activity_date :date
|
7
|
+
# group_name :string(255)
|
8
|
+
# option_name :string(255)
|
9
|
+
# conversion_value :decimal(8, 2)
|
10
|
+
# views :integer
|
11
|
+
# conversions :integer
|
12
|
+
#
|
13
|
+
|
14
|
+
class Conductor::DailyExperiment < ActiveRecord::Base
|
15
|
+
set_table_name "conductor_daily_experiments"
|
16
|
+
named_scope :since, lambda { |a_date| { :conditions => ['activity_date >= ?',a_date] }}
|
17
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# == Schema Information
|
2
|
+
#
|
3
|
+
# Table name: conductor_raw_experiments
|
4
|
+
#
|
5
|
+
# id :integer not null, primary key
|
6
|
+
# identity_id :string(255)
|
7
|
+
# group_name :string(255)
|
8
|
+
# option_name :string(255)
|
9
|
+
# conversion_value :decimal(8, 2)
|
10
|
+
# created_at :datetime
|
11
|
+
# updated_at :datetime
|
12
|
+
#
|
13
|
+
|
14
|
+
class Conductor::RawExperiment < ActiveRecord::Base
|
15
|
+
set_table_name "conductor_raw_experiments"
|
16
|
+
|
17
|
+
validates_presence_of :group_name, :option_name
|
18
|
+
|
19
|
+
def created_date
|
20
|
+
self.created_at.strftime('%Y-%m-%d')
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# == Schema Information
|
2
|
+
#
|
3
|
+
# Table name: conductor_weight_histories
|
4
|
+
#
|
5
|
+
# id :integer not null, primary key
|
6
|
+
# group_name :string(255)
|
7
|
+
# option_name :string(255)
|
8
|
+
# weight :decimal(8, 2)
|
9
|
+
# computed_at :datetime
|
10
|
+
#
|
11
|
+
|
12
|
+
class Conductor::WeightHistory < ActiveRecord::Base
|
13
|
+
set_table_name "conductor_weight_histories"
|
14
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# == Schema Information
|
2
|
+
#
|
3
|
+
# Table name: conductor_weighted_experiments
|
4
|
+
#
|
5
|
+
# id :integer not null, primary key
|
6
|
+
# group_name :string(255)
|
7
|
+
# option_name :string(255)
|
8
|
+
# weight :decimal(8, 2)
|
9
|
+
# created_at :datetime
|
10
|
+
# updated_at :datetime
|
11
|
+
#
|
12
|
+
|
13
|
+
class Conductor::WeightedExperiment < ActiveRecord::Base
|
14
|
+
set_table_name "conductor_weighted_experiments"
|
15
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class Conductor
|
2
|
+
class RollUp
|
3
|
+
def self.process
|
4
|
+
Conductor::RawExperiment.all.group_by(&:created_date).each do |day, daily_rows|
|
5
|
+
|
6
|
+
# remove all the existing data for that day
|
7
|
+
Conductor::DailyExperiment.delete_all(:activity_date => day)
|
8
|
+
|
9
|
+
daily_rows.group_by(&:group_name).each do |group_name, group_rows|
|
10
|
+
group_rows.group_by(&:option_name).each do |option_name, option_rows|
|
11
|
+
conversion_value = option_rows.select {|x| !x.conversion_value.nil?}.inject(0) {|res, x| res += x.conversion_value}
|
12
|
+
views = option_rows.count
|
13
|
+
conversions = option_rows.count {|x| !x.conversion_value.nil?}
|
14
|
+
Conductor::DailyExperiment.create!(:activity_date => day,
|
15
|
+
:group_name => group_name,
|
16
|
+
:option_name => option_name,
|
17
|
+
:conversion_value => conversion_value,
|
18
|
+
:views => views,
|
19
|
+
:conversions => conversions )
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
class Conductor
|
2
|
+
class Weights
|
3
|
+
class << self
|
4
|
+
def compute
|
5
|
+
Conductor::WeightedExperiment.delete_all # => remove all old data
|
6
|
+
|
7
|
+
# loop through each group and determing weight of options
|
8
|
+
Conductor::DailyExperiment.since(14.days.ago).group_by(&:group_name).each do |group_name, group_rows|
|
9
|
+
total = group_rows.sum_it(:conversion_value)
|
10
|
+
data = total ? compute_weights_for_group(group_name, group_rows, total) : assign_equal_weights(group_rows)
|
11
|
+
update_weights_in_db(group_name, data)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
# loops through all the options for a given group and computes the weights for
|
18
|
+
# each alternative
|
19
|
+
def compute_weights_for_group(group_name, group_rows, total)
|
20
|
+
Conductor.log('compute_weights_for_group')
|
21
|
+
|
22
|
+
data = []
|
23
|
+
recently_launched = []
|
24
|
+
max_weight = 0
|
25
|
+
|
26
|
+
group_rows.group_by(&:option_name).each do |option_name, option_rows|
|
27
|
+
first_found_date = option_rows.map(&:activity_date).sort.first
|
28
|
+
days_ago = Date.today - first_found_date
|
29
|
+
|
30
|
+
if days_ago >= MINIMUM_LAUNCH_DAYS
|
31
|
+
data << compute_weight_for_option(option_name, option_rows, max_weight, total)
|
32
|
+
else
|
33
|
+
Conductor.log("adding #{option_name} to recently launched array")
|
34
|
+
recently_launched << {:name => option_name, :days_ago => days_ago}
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
data += weight_recently_launched(max_weight, recently_launched)
|
39
|
+
return data
|
40
|
+
end
|
41
|
+
|
42
|
+
def compute_weight_for_option(option_name, option_rows, max_weight, total)
|
43
|
+
Conductor.log("compute_weight_for_option for #{option_name}")
|
44
|
+
|
45
|
+
aggregates = {:name => option_name}
|
46
|
+
|
47
|
+
weight = option_rows.sum_it(:conversion_value) / total
|
48
|
+
max_weight = weight if weight > max_weight
|
49
|
+
aggregates.merge!({:weight => weight})
|
50
|
+
|
51
|
+
return aggregates
|
52
|
+
end
|
53
|
+
|
54
|
+
def weight_recently_launched(max_weight, recently_launched)
|
55
|
+
# loop through recently_launched to create weights for table
|
56
|
+
# the handicap sets a newly launched item to the max weight * MAX_WEIGHTING_FACTOR and then
|
57
|
+
# slowly lowers its power until the launch period is over
|
58
|
+
data = []
|
59
|
+
max_weight = 0 ? 1 : max_weight # => if a max weight could not be computed, set it to 1
|
60
|
+
Conductor.log("max weight: #{max_weight}")
|
61
|
+
recently_launched.each do |option|
|
62
|
+
handicap = (option[:days_ago].to_f / MINIMUM_LAUNCH_DAYS)
|
63
|
+
launch_window = (MINIMUM_LAUNCH_DAYS - option[:days_ago]) if MINIMUM_LAUNCH_DAYS > option[:days_ago]
|
64
|
+
Conductor.log("Handicap for #{option[:name]} is #{handicap} (#{option[:days_ago]} days ago)")
|
65
|
+
data << {:name => option[:name], :weight => max_weight * MAX_WEIGHTING_FACTOR * (1 - handicap), :launch_window => launch_window}
|
66
|
+
end
|
67
|
+
data
|
68
|
+
end
|
69
|
+
|
70
|
+
def assign_equal_weights(group_rows)
|
71
|
+
Conductor.log('assign_equal_weights')
|
72
|
+
|
73
|
+
# weight everything the same since there were no conversions
|
74
|
+
data = []
|
75
|
+
group_rows.group_by(&:option_name).each do |option_name, option_rows|
|
76
|
+
data << {:name => option_name, :weight => 1}
|
77
|
+
end
|
78
|
+
data
|
79
|
+
end
|
80
|
+
|
81
|
+
# creates new records in weights table and adds weights to weight history for reporting
|
82
|
+
def update_weights_in_db(group_name, data)
|
83
|
+
data.each { |option|
|
84
|
+
Conductor::WeightedExperiment.create!(:group_name => group_name, :option_name => option[:name], :weight => option[:weight])
|
85
|
+
Conductor::WeightHistory.create!(:group_name => group_name, :option_name => option[:name], :weight => option[:weight], :launch_window => option[:launch_window], :computed_at => Time.now)
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
data/test/test_conductor.rb
CHANGED
@@ -1,7 +1,225 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
3
|
class TestConductor < Test::Unit::TestCase
|
4
|
-
|
5
|
-
|
6
|
-
|
4
|
+
#Wipes cache, D/B prior to doing a test run.
|
5
|
+
VectorSixteen.cache.clear
|
6
|
+
|
7
|
+
test "will automatically assign an identity if none is specified" do
|
8
|
+
assert VectorSixteen.identity != nil
|
9
|
+
end
|
10
|
+
|
11
|
+
test "will select one of the specified options randomly" do
|
12
|
+
selected = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"]) # => value must be unique
|
13
|
+
assert ["a", "b", "c"].include? selected
|
14
|
+
end
|
15
|
+
|
16
|
+
test "will always select the same option using the cache" do
|
17
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
18
|
+
|
19
|
+
selected = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"]) # => value must be unique
|
20
|
+
different = false
|
21
|
+
|
22
|
+
(1..100).each do |x|
|
23
|
+
different = true if selected != VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"])
|
24
|
+
end
|
25
|
+
|
26
|
+
assert !different
|
27
|
+
end
|
28
|
+
|
29
|
+
test "will select a lander and then successfully record a conversion" do
|
30
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
31
|
+
selected = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"]) # => value must be unique
|
32
|
+
|
33
|
+
VectorSixteen::Experiment.track!
|
34
|
+
|
35
|
+
experiments = V16::RawExperiment.find_all_by_identity_id(VectorSixteen.identity)
|
36
|
+
assert_equal 1, experiments.count
|
37
|
+
assert_equal 1, experiments.first.conversion_value
|
38
|
+
end
|
39
|
+
|
40
|
+
test "will select a lander and then successfully record custom conversion value" do
|
41
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
42
|
+
selected = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"]) # => value must be unique
|
43
|
+
|
44
|
+
VectorSixteen::Experiment.track!({:value => 12.34})
|
45
|
+
|
46
|
+
experiments = V16::RawExperiment.find_all_by_identity_id(VectorSixteen.identity)
|
47
|
+
assert_equal 1, experiments.count
|
48
|
+
assert_equal 12.34, experiments.first.conversion_value
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
test "will record three different experiments with two goals but a single conversion for all goals for the same identity" do
|
54
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
55
|
+
first = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"], {:goal => 'goal_1'}) # => value must be unique
|
56
|
+
second = VectorSixteen::Experiment.pick('b_group', ["1", "2", "3"], {:goal => 'goal_2'}) # => value must be unique
|
57
|
+
third = VectorSixteen::Experiment.pick('c_group', ["zz", "xx", "yy"], {:goal => 'goal_1'}) # => value must be unique
|
58
|
+
|
59
|
+
VectorSixteen::Experiment.track!
|
60
|
+
|
61
|
+
experiments = V16::RawExperiment.find_all_by_identity_id(VectorSixteen.identity)
|
62
|
+
assert_equal 3, experiments.count
|
63
|
+
assert_equal 2, experiments.count {|x| x.goal == 'goal_1'}
|
64
|
+
assert_equal 3, experiments.sum_it(:conversion_value)
|
65
|
+
end
|
66
|
+
|
67
|
+
test "will record three different experiments with two goals but only track a conversion for goal_1" do
|
68
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
69
|
+
first = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"], {:goal => 'goal_1'}) # => value must be unique
|
70
|
+
second = VectorSixteen::Experiment.pick('b_group', ["1", "2", "3"], {:goal => 'goal_2'}) # => value must be unique
|
71
|
+
third = VectorSixteen::Experiment.pick('c_group', ["zz", "xx", "yy"], {:goal => 'goal_1'}) # => value must be unique
|
72
|
+
|
73
|
+
VectorSixteen::Experiment.track!({:goal => 'goal_1'})
|
74
|
+
|
75
|
+
experiments = V16::RawExperiment.find_all_by_identity_id(VectorSixteen.identity)
|
76
|
+
assert_equal 3, experiments.count
|
77
|
+
assert_equal 2, experiments.count {|x| x.goal == 'goal_1'}
|
78
|
+
assert_equal 2, experiments.sum_it(:conversion_value)
|
79
|
+
end
|
80
|
+
|
81
|
+
|
82
|
+
test "will almost equally select each option if no weights exist" do
|
83
|
+
a = 0
|
84
|
+
b = 0
|
85
|
+
c = 0
|
86
|
+
(1..1000).each do |x|
|
87
|
+
selected_lander = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"]) # => value must be unique
|
88
|
+
case selected_lander
|
89
|
+
when 'a' then
|
90
|
+
a += 1
|
91
|
+
when 'b' then
|
92
|
+
b += 1
|
93
|
+
when 'c' then
|
94
|
+
c += 1
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
nums = [] << a << b << c
|
99
|
+
nums.sort!
|
100
|
+
range = nums.last - nums.first
|
101
|
+
|
102
|
+
assert (nums.first * 0.20) >= range
|
103
|
+
end
|
104
|
+
|
105
|
+
test "will correctly RollUp daily data" do
|
106
|
+
# seed
|
107
|
+
seed_raw_data(100)
|
108
|
+
|
109
|
+
# rollup
|
110
|
+
VectorSixteen::RollUp.process
|
111
|
+
|
112
|
+
# do some checks
|
113
|
+
assert V16::DailyExperiment.count > 2
|
114
|
+
assert V16::DailyExperiment.all.detect {|x| x.conversions > 0}
|
115
|
+
assert V16::DailyExperiment.all.detect {|x| x.views > 0}
|
116
|
+
assert V16::DailyExperiment.all.detect {|x| x.conversion_value > 0}
|
117
|
+
end
|
118
|
+
|
119
|
+
test "will correctly populate weighting table" do
|
120
|
+
# seed
|
121
|
+
seed_raw_data(100)
|
122
|
+
|
123
|
+
# rollup
|
124
|
+
VectorSixteen::RollUp.process
|
125
|
+
|
126
|
+
# compute weights
|
127
|
+
VectorSixteen::Weights.compute
|
128
|
+
end
|
129
|
+
|
130
|
+
test "will populate the weighting table with equal weights if all new options are launched" do
|
131
|
+
wipe
|
132
|
+
seed_raw_data(100, 7)
|
133
|
+
|
134
|
+
# rollup
|
135
|
+
VectorSixteen::RollUp.process
|
136
|
+
|
137
|
+
# compute weights
|
138
|
+
VectorSixteen::Weights.compute
|
139
|
+
|
140
|
+
# this makes the following assumptions:
|
141
|
+
# MINIMUM_LAUNCH_DAYS = 7
|
142
|
+
# each weight will be equal to 0.18
|
143
|
+
assert_equal 0.54, V16::WeightedExperiment.all.sum_it(:weight).to_f
|
144
|
+
end
|
145
|
+
|
146
|
+
test "will populate the weighting table with different weights" do
|
147
|
+
wipe
|
148
|
+
seed_raw_data(100, 14)
|
149
|
+
|
150
|
+
# rollup
|
151
|
+
VectorSixteen::RollUp.process
|
152
|
+
|
153
|
+
# compute weights
|
154
|
+
VectorSixteen::Weights.compute
|
155
|
+
|
156
|
+
# if this DOES NOT work then each weight will be equal to 0.18
|
157
|
+
assert_not_equal 0.54, V16::WeightedExperiment.all.sum_it(:weight).to_f
|
158
|
+
end
|
159
|
+
|
160
|
+
test "will record the new weights in the weight history table in database" do
|
161
|
+
wipe
|
162
|
+
seed_raw_data(100, 14)
|
163
|
+
|
164
|
+
# rollup
|
165
|
+
VectorSixteen::RollUp.process
|
166
|
+
|
167
|
+
# compute weights
|
168
|
+
VectorSixteen::Weights.compute
|
169
|
+
|
170
|
+
assert V16::WeightHistory.count > 1
|
171
|
+
end
|
172
|
+
|
173
|
+
test "will correctly record the launch window in the weight histories table" do
|
174
|
+
wipe
|
175
|
+
seed_raw_data(10, 6)
|
176
|
+
|
177
|
+
# rollup
|
178
|
+
VectorSixteen::RollUp.process
|
179
|
+
|
180
|
+
# compute weights
|
181
|
+
VectorSixteen::Weights.compute
|
182
|
+
|
183
|
+
# make sure that launch_window values can be detected
|
184
|
+
assert_not_nil V16::WeightHistory.find(:all, :conditions => 'launch_window > 0')
|
185
|
+
end
|
186
|
+
|
187
|
+
test "will return a weight 1.25 times higher than the highest weight for a newly launched and non-recorded alernative" do
|
188
|
+
wipe
|
189
|
+
seed_raw_data(100, 14)
|
190
|
+
|
191
|
+
# rollup
|
192
|
+
VectorSixteen::RollUp.process
|
193
|
+
|
194
|
+
# compute weights
|
195
|
+
VectorSixteen::Weights.compute
|
196
|
+
|
197
|
+
# get the highest weight
|
198
|
+
max_weight = V16::WeightedExperiment.maximum(:weight)
|
199
|
+
|
200
|
+
# pick something
|
201
|
+
weights = VectorSixteen::Experiment.weights('a_group', ["a", "b", "c", "f"]) # => value must be unique
|
202
|
+
|
203
|
+
assert_equal weights['f'], (max_weight * 1.25)
|
204
|
+
end
|
205
|
+
|
206
|
+
private
|
207
|
+
|
208
|
+
def wipe
|
209
|
+
V16::DailyExperiment.delete_all
|
210
|
+
V16::RawExperiment.delete_all
|
211
|
+
V16::WeightedExperiment.delete_all
|
212
|
+
V16::WeightHistory.delete_all
|
213
|
+
end
|
214
|
+
|
215
|
+
def seed_raw_data(num, days_ago=14)
|
216
|
+
# seed the raw data
|
217
|
+
(1..num).each do |x|
|
218
|
+
VectorSixteen.identity = ActiveSupport::SecureRandom.hex(16)
|
219
|
+
|
220
|
+
options = {:created_at => rand(days_ago).days.ago}
|
221
|
+
options.merge!({:conversion_value => rand(100)}) if rand() < 0.20 # => convert 20% of traffic
|
222
|
+
selected_lander = VectorSixteen::Experiment.pick('a_group', ["a", "b", "c"], options) # => value must be unique
|
223
|
+
end
|
224
|
+
end
|
7
225
|
end
|
metadata
CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 0
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 0.
|
7
|
+
- 2
|
8
|
+
- 9
|
9
|
+
version: 0.2.9
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Noctivity
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-09-
|
17
|
+
date: 2010-09-27 00:00:00 -04:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|
@@ -28,14 +28,21 @@ extra_rdoc_files:
|
|
28
28
|
- LICENSE
|
29
29
|
- README.rdoc
|
30
30
|
files:
|
31
|
-
- .document
|
32
|
-
- .gitignore
|
33
31
|
- LICENSE
|
34
32
|
- README.rdoc
|
35
33
|
- Rakefile
|
36
34
|
- VERSION
|
37
|
-
-
|
35
|
+
- generators/conductor_migration/conductor_migration_generator.rb
|
36
|
+
- generators/conductor_migration/templates/conductor_migration.rb
|
37
|
+
- init.rb
|
38
38
|
- lib/conductor.rb
|
39
|
+
- lib/conductor/experiment.rb
|
40
|
+
- lib/conductor/models/daily_experiment.rb
|
41
|
+
- lib/conductor/models/raw_experiment.rb
|
42
|
+
- lib/conductor/models/weight_history.rb
|
43
|
+
- lib/conductor/models/weighted_experiment.rb
|
44
|
+
- lib/conductor/roll_up.rb
|
45
|
+
- lib/conductor/weights.rb
|
39
46
|
- test/helper.rb
|
40
47
|
- test/test_conductor.rb
|
41
48
|
has_rdoc: true
|
@@ -69,7 +76,7 @@ rubyforge_project: conductor
|
|
69
76
|
rubygems_version: 1.3.7
|
70
77
|
signing_key:
|
71
78
|
specification_version: 3
|
72
|
-
summary:
|
79
|
+
summary: lets you just try things while always maximizing towards a goal (e.g. purchase, signups, etc)
|
73
80
|
test_files:
|
74
81
|
- test/helper.rb
|
75
82
|
- test/test_conductor.rb
|
data/.document
DELETED
data/.gitignore
DELETED
data/conductor.gemspec
DELETED
@@ -1,52 +0,0 @@
|
|
1
|
-
# Generated by jeweler
|
2
|
-
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
-
# -*- encoding: utf-8 -*-
|
5
|
-
|
6
|
-
Gem::Specification.new do |s|
|
7
|
-
s.name = %q{conductor}
|
8
|
-
s.version = "0.0.0"
|
9
|
-
|
10
|
-
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
-
s.authors = ["Noctivity"]
|
12
|
-
s.date = %q{2010-09-26}
|
13
|
-
s.description = %q{Conductor is the bastard child of a/b testing and personalization. It throws everything you know about creating a web site our the window and lets you just "try stuff" without ever having to worry about not maximing your site's "purpose." Have a new landing page? Just throw it to the conductor. Want to try different price points - conductor. Different form designs? Conductor. Conductor will rotate all alternatives through the mix and eventually settle on the top performing of all, without you having to do anything other than just creating. Think "intelligent A/B testing" on steriods.}
|
14
|
-
s.email = %q{jlippiner@noctivity.com}
|
15
|
-
s.extra_rdoc_files = [
|
16
|
-
"LICENSE",
|
17
|
-
"README.rdoc"
|
18
|
-
]
|
19
|
-
s.files = [
|
20
|
-
".document",
|
21
|
-
".gitignore",
|
22
|
-
"LICENSE",
|
23
|
-
"README.rdoc",
|
24
|
-
"Rakefile",
|
25
|
-
"VERSION",
|
26
|
-
"conductor.gemspec",
|
27
|
-
"lib/conductor.rb",
|
28
|
-
"test/helper.rb",
|
29
|
-
"test/test_conductor.rb"
|
30
|
-
]
|
31
|
-
s.homepage = %q{http://github.com/noctivityinc/conductor}
|
32
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
33
|
-
s.require_paths = ["lib"]
|
34
|
-
s.rubyforge_project = %q{conductor}
|
35
|
-
s.rubygems_version = %q{1.3.7}
|
36
|
-
s.summary = %q{let's you just try things while always maximizing towards a goal (e.g. purchase, signups, etc)}
|
37
|
-
s.test_files = [
|
38
|
-
"test/helper.rb",
|
39
|
-
"test/test_conductor.rb"
|
40
|
-
]
|
41
|
-
|
42
|
-
if s.respond_to? :specification_version then
|
43
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
44
|
-
s.specification_version = 3
|
45
|
-
|
46
|
-
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
47
|
-
else
|
48
|
-
end
|
49
|
-
else
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|