bandit 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +9 -5
- data/Rakefile +1 -0
- data/bandit.gemspec +2 -1
- data/lib/bandit/experiment.rb +2 -2
- data/lib/bandit/extensions/array.rb +3 -0
- data/lib/bandit/memoizable.rb +17 -0
- data/lib/bandit/players/epsilon_greedy.rb +1 -1
- data/lib/bandit/players/round_robin.rb +1 -1
- data/lib/bandit/storage/base.rb +10 -0
- data/lib/bandit/storage/memcache.rb +16 -6
- data/lib/bandit/storage/redis.rb +17 -7
- data/lib/bandit/version.rb +1 -1
- data/lib/bandit.rb +14 -1
- data/lib/generators/bandit/USAGE +1 -1
- data/players.rdoc +1 -2
- data/test/redis_storage_test.rb +1 -1
- data/whybandit.rdoc +31 -0
- metadata +25 -7
data/README.rdoc
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
= bandit
|
2
2
|
|
3
|
-
Bandit is a multi-armed bandit
|
3
|
+
Bandit is a multi-armed bandit optimization framework for Rails. It provides an alternative to A/B testing in Rails. For background and a comparison with A/B testing, see the whybandit.rdoc document.
|
4
4
|
|
5
5
|
= Installation
|
6
6
|
First, add the following to your Gemfile in your Rails 3 app:
|
@@ -42,7 +42,7 @@ You can force a particular alternative by adding a query parameter named "bandit
|
|
42
42
|
will then force the alternative to be "40".
|
43
43
|
|
44
44
|
== Controller
|
45
|
-
To track a
|
45
|
+
To track a conversion in your controller:
|
46
46
|
|
47
47
|
bandit_convert! :click_test
|
48
48
|
|
@@ -65,7 +65,11 @@ To run tests:
|
|
65
65
|
|
66
66
|
To produce fake data for the past week, first create an experiment definition. Then, run the following rake task:
|
67
67
|
|
68
|
-
rake bandit:populate_data[experiment_name]
|
68
|
+
rake bandit:populate_data[<experiment_name>]
|
69
69
|
|
70
|
-
|
71
|
-
|
70
|
+
For instance, to generate a week's worth of fake data for the click_test above:
|
71
|
+
|
72
|
+
rake bandit:populate_data[click_test]
|
73
|
+
|
74
|
+
= Fault Tolerance
|
75
|
+
If the storage mechanism fails, then Bandit will automatically switch to in memory storage. It will then check every 5 minutes after that to see if the original storage mechanism is back up. If you have distributed front ends then each front end will continue to optimize (based on the in memory storage), but this optimization will be inefficient compared to shared storage among all front ends.
|
data/Rakefile
CHANGED
data/bandit.gemspec
CHANGED
@@ -17,5 +17,6 @@ Gem::Specification.new do |s|
|
|
17
17
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
s.add_dependency("rails", ">= 3.0.5")
|
20
|
-
s.add_dependency("
|
20
|
+
s.add_dependency("redis", "= 2.2.2")
|
21
|
+
s.add_development_dependency('rdoc')
|
21
22
|
end
|
data/lib/bandit/experiment.rb
CHANGED
@@ -20,8 +20,8 @@ module Bandit
|
|
20
20
|
@@instances
|
21
21
|
end
|
22
22
|
|
23
|
-
def choose(default
|
24
|
-
if
|
23
|
+
def choose(default=nil)
|
24
|
+
if default && alternatives.include?(default)
|
25
25
|
alt = default
|
26
26
|
else
|
27
27
|
alt = Bandit.player.choose_alternative(self)
|
data/lib/bandit/memoizable.rb
CHANGED
@@ -11,5 +11,22 @@ module Bandit
|
|
11
11
|
end
|
12
12
|
@memoized[key]
|
13
13
|
end
|
14
|
+
|
15
|
+
module ClassMethods
|
16
|
+
def memoize_method(method, time=60)
|
17
|
+
original_method = "unmemoized_#{method}_#{Time.now.to_i}"
|
18
|
+
alias_method original_method, method
|
19
|
+
module_eval(<<-EVAL, __FILE__, __LINE__)
|
20
|
+
def #{method}(*args, &block)
|
21
|
+
memoize(:#{original_method}, #{time}) { send(:#{original_method}, *args, &block) }
|
22
|
+
end
|
23
|
+
EVAL
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.included(base)
|
28
|
+
base.extend(ClassMethods)
|
29
|
+
end
|
30
|
+
|
14
31
|
end
|
15
32
|
end
|
data/lib/bandit/storage/base.rb
CHANGED
@@ -120,5 +120,15 @@ module Bandit
|
|
120
120
|
def make_key(parts)
|
121
121
|
parts.join(":")
|
122
122
|
end
|
123
|
+
|
124
|
+
def with_failure_grace(fail_default=0)
|
125
|
+
begin
|
126
|
+
yield
|
127
|
+
rescue
|
128
|
+
Bandit.storage_failed!
|
129
|
+
Rails.logger.error "Storage method #{self.class} failed. Falling back to memory storage."
|
130
|
+
fail_default
|
131
|
+
end
|
132
|
+
end
|
123
133
|
end
|
124
134
|
end
|
@@ -9,26 +9,36 @@ module Bandit
|
|
9
9
|
# increment key by count
|
10
10
|
def incr(key, count=1)
|
11
11
|
# memcache incr seems to be broken in memcache-client gem
|
12
|
-
|
12
|
+
with_failure_grace(count) {
|
13
|
+
set(key, get(key, 0) + count)
|
14
|
+
}
|
13
15
|
end
|
14
16
|
|
15
17
|
# initialize key if not set
|
16
|
-
def init(key, value)
|
17
|
-
|
18
|
+
def init(key, value)
|
19
|
+
with_failure_grace(value) {
|
20
|
+
@memcache.add(key, value)
|
21
|
+
}
|
18
22
|
end
|
19
23
|
|
20
24
|
# get key if exists, otherwise 0
|
21
25
|
def get(key, default=0)
|
22
|
-
|
26
|
+
with_failure_grace(default) {
|
27
|
+
@memcache.get(key) || default
|
28
|
+
}
|
23
29
|
end
|
24
30
|
|
25
31
|
# set key with value, regardless of whether it is set or not
|
26
32
|
def set(key, value)
|
27
|
-
|
33
|
+
with_failure_grace(value) {
|
34
|
+
@memcache.set(key, value)
|
35
|
+
}
|
28
36
|
end
|
29
37
|
|
30
38
|
def clear!
|
31
|
-
|
39
|
+
with_failure_grace(nil) {
|
40
|
+
@memcache.flush_all
|
41
|
+
}
|
32
42
|
end
|
33
43
|
end
|
34
44
|
end
|
data/lib/bandit/storage/redis.rb
CHANGED
@@ -10,28 +10,38 @@ module Bandit
|
|
10
10
|
|
11
11
|
# increment key by count
|
12
12
|
def incr(key, count=1)
|
13
|
-
|
13
|
+
with_failure_grace(count) {
|
14
|
+
@redis.incrby(key, count)
|
15
|
+
}
|
14
16
|
end
|
15
17
|
|
16
18
|
# initialize key if not set
|
17
19
|
def init(key, value)
|
18
|
-
|
20
|
+
with_failure_grace(value) {
|
21
|
+
@redis.set(key, value) if get(key, nil).nil?
|
22
|
+
}
|
19
23
|
end
|
20
24
|
|
21
25
|
# get key if exists, otherwise 0
|
22
26
|
def get(key, default=0)
|
23
|
-
|
24
|
-
|
25
|
-
|
27
|
+
with_failure_grace(default) {
|
28
|
+
val = @redis.get(key)
|
29
|
+
return default if val.nil?
|
30
|
+
val.numeric? ? val.to_i : val
|
31
|
+
}
|
26
32
|
end
|
27
33
|
|
28
34
|
# set key with value, regardless of whether it is set or not
|
29
35
|
def set(key, value)
|
30
|
-
|
36
|
+
with_failure_grace(value) {
|
37
|
+
@redis.set(key, value)
|
38
|
+
}
|
31
39
|
end
|
32
40
|
|
33
41
|
def clear!
|
34
|
-
|
42
|
+
with_failure_grace(nil) {
|
43
|
+
@redis.flushdb
|
44
|
+
}
|
35
45
|
end
|
36
46
|
end
|
37
47
|
end
|
data/lib/bandit/version.rb
CHANGED
data/lib/bandit.rb
CHANGED
@@ -15,11 +15,14 @@ require "bandit/storage/memcache"
|
|
15
15
|
require "bandit/storage/redis"
|
16
16
|
|
17
17
|
require "bandit/extensions/controller_concerns"
|
18
|
+
require "bandit/extensions/array"
|
18
19
|
require "bandit/extensions/view_concerns"
|
19
20
|
require "bandit/extensions/time"
|
20
21
|
require "bandit/extensions/string"
|
21
22
|
|
22
23
|
module Bandit
|
24
|
+
@@storage_failure_at = nil
|
25
|
+
|
23
26
|
def self.config
|
24
27
|
@config ||= Config.new
|
25
28
|
end
|
@@ -32,13 +35,23 @@ module Bandit
|
|
32
35
|
end
|
33
36
|
|
34
37
|
def self.storage
|
35
|
-
|
38
|
+
# try using configured storage at least once every 5 minutes until resolved
|
39
|
+
if @@storage_failure_at.nil? or (Time.now.to_i - @@storage_failure_at) > 300
|
40
|
+
@storage ||= BaseStorage.get_storage(Bandit.config.storage.intern, Bandit.config.storage_config)
|
41
|
+
else
|
42
|
+
Rails.logger.warn "storage failure detected #{Time.now.to_i - @@storage_failure_at} seconds ago - using memory storage for 5 minutes"
|
43
|
+
BaseStorage.get_storage(:memory, Bandit.config.storage_config)
|
44
|
+
end
|
36
45
|
end
|
37
46
|
|
38
47
|
def self.player
|
39
48
|
@player ||= BasePlayer.get_player(Bandit.config.player.intern, Bandit.config.player_config)
|
40
49
|
end
|
41
50
|
|
51
|
+
def self.storage_failed!
|
52
|
+
@@storage_failure_at = Time.now.to_i
|
53
|
+
end
|
54
|
+
|
42
55
|
def self.get_experiment(name)
|
43
56
|
exp = Experiment.instances.select { |e| e.name == name }
|
44
57
|
exp.length > 0 ? exp.first : nil
|
data/lib/generators/bandit/USAGE
CHANGED
data/players.rdoc
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
= Bandit Players
|
2
|
-
There are a number of different possible players that each seek to explore/exploit using different methods.
|
2
|
+
There are a number of different possible players that each seek to explore/exploit using different methods. Each can be configured in the *bandit.yml* file in the config directory under your *Rails.root*.
|
3
3
|
|
4
4
|
== Epsilon Greedy
|
5
5
|
The epsilon greedy player selects the best alternative with a probability of 1 - epsilon, and selects uniformly among the other alternatives with a probability of epsilon. You can set the value of epsilon in the config file like this:
|
@@ -15,7 +15,6 @@ The epsilon greedy player selects the best alternative with a probability of 1 -
|
|
15
15
|
== Round Robin
|
16
16
|
This is included for testing purposes only. It will not optimize. There are no config options.
|
17
17
|
|
18
|
-
|
19
18
|
production:
|
20
19
|
...
|
21
20
|
player: round_robin
|
data/test/redis_storage_test.rb
CHANGED
data/whybandit.rdoc
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
= Bandit vs A/B Testing
|
2
|
+
In a typical A/B test, two alternatives are compared to see which produces the most "conversions" (that is, desired results). For instance, if you have a website with a big "Sign Up" button that you want visitors to click, you may wish to choose different background colors. Under typical A/B testing guildlines, you would pick a number (say, *N*) of users for a test and show half of them one color and half of them another color. After users are shown the button, you record the number of clicks that result from viewing each color. Once *N* users view one of the two alternatives, a statistical test (generally categorical, like a Chi-Square Test or a G-Test) is run to determine whether or not the number of clicks (aka, "conversions") for one color were higher than the number of clicks for the other color. This test determines whether the difference you observed was likely due simply to chance or whether the difference you saw was more likely due to an actual difference in the rate of conversion.
|
3
|
+
|
4
|
+
This method of testing is popular, but is fraught with issues (practical and statistical). Bandit provides an implementation of an alternative method of testing that solves many of these issues.
|
5
|
+
|
6
|
+
== Issues with A/B Testing
|
7
|
+
There are a number of issues with A/B testing (some of which have been described in more detail here[http://untyped.com/untyping/2011/02/11/stop-ab-testing-and-make-out-like-a-bandit]):
|
8
|
+
|
9
|
+
1. You can't try anything too crazy without having to worry about half of your users not converting. For instance, you may want to try a horrendous color for your "Buy Now" button but are too afraid about potentially harming sales if your users hate it. In this case, the risk of a big change may outweigh the possible benefit if your users like it.
|
10
|
+
1. A/B testing provides a way of only testing two alternatives at once. Pick two, wait, pick two more, wait - this is not the easiest workflow if you want to test 50 options.
|
11
|
+
1. With A/B Testing, you need to have a fixed sample size to make the test valid (otherwise, you run the risk of repeated significance testing errors, as described in more detail here[http://www.evanmiller.org/how-not-to-run-an-ab-test.html]).
|
12
|
+
1. Due to the fixed sample size requirement, you may have to wait a while before you get any results from your test (especially if the expected improvement is marginal, in which case your sample size would need to be larger). This problem can be compounded if you don't get much traffic.
|
13
|
+
1. Designers and developers don't want to (and shouldn't have to) understand statistical concepts like power[http://en.wikipedia.org/wiki/Statistical_power], p-values[http://en.wikipedia.org/wiki/P-value], or confidence[http://en.wikipedia.org/wiki/Confidence_intervals] when creating and evaluating tests.
|
14
|
+
1. There are no guidelines for what you should do when A performs just as well as B.
|
15
|
+
|
16
|
+
== The Bandit Method
|
17
|
+
The ultimate goal of A/B testing is to increase conversions. The problem can be described terms that differ greatly from the multitude of questions A/B testing brings (i.e., "Is A better than B?" followed by "Is B better than C?" followed by "Is C better than D?" _ad_ _infinitum_). Instead, imagine you have a multitude of possible alternatives, and you want to make a decent choice between alternatives you know perform well and alternatives you haven't tried very often each time a user requests a page. With each page load, pick the best alternative most of the time and an alternative that hasn't been displayed much some of the time. After each display, monitor the conversions and update what you consider the "better" alternatives to be. This is the basic method of a solution to what is called the multi-armed bandit problem.
|
18
|
+
|
19
|
+
With a bandit solution, there is no concept of a "test". At no point does the system announce a winner and a loser. Alternatives can be added or removed at any time. The better performing alternatives will be displayed more often, and the worst alternatives will rarely be displayed. At any point, if one of the poorly performing alternatives begins to perform better it will be shown more often. This provides solutions to all of the problems listed above:
|
20
|
+
|
21
|
+
1. Go ahead and try something crazy. If it performs poorly, it won't be shown very often.
|
22
|
+
1. Pick as many alternatives as you'd like and add them.
|
23
|
+
1. There's no "test", and no minimal sample size needed before optimization can start.
|
24
|
+
1. Information about conversions is utilized as users convert or do not convert. There is no pause before results can be immediately used in selecting the next alternative to display to a visitor.
|
25
|
+
1. Designers and developers can add alternatives or remove them at any time. The system will adjust immediately. If an alternative seems to be consistently performing poorly, it can be removed at any time. Alternatively, it can just be left forever. The best option will always be displayed the most often. There are no complicated decisions that have to be made up front or requirements that designers or developers know anything about proper statistical hypothesis testing.
|
26
|
+
1. If one alternative performs the same as another, they will both be displayed with the same regularity. There would be no need to choose one over the other or remove either of them.
|
27
|
+
|
28
|
+
== Reference
|
29
|
+
* http://untyped.com/untyping/2011/02/11/stop-ab-testing-and-make-out-like-a-bandit
|
30
|
+
* http://en.wikipedia.org/wiki/Multi-armed_bandit
|
31
|
+
* http://www.evanmiller.org/how-not-to-run-an-ab-test.html
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bandit
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
9
|
+
- 4
|
10
|
+
version: 0.0.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Brian Muller
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-11-06 01:00:00 -04:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -35,9 +35,25 @@ dependencies:
|
|
35
35
|
type: :runtime
|
36
36
|
version_requirements: *id001
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: redis
|
39
39
|
prerelease: false
|
40
40
|
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - "="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 3
|
46
|
+
segments:
|
47
|
+
- 2
|
48
|
+
- 2
|
49
|
+
- 2
|
50
|
+
version: 2.2.2
|
51
|
+
type: :runtime
|
52
|
+
version_requirements: *id002
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: rdoc
|
55
|
+
prerelease: false
|
56
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
57
|
none: false
|
42
58
|
requirements:
|
43
59
|
- - ">="
|
@@ -46,8 +62,8 @@ dependencies:
|
|
46
62
|
segments:
|
47
63
|
- 0
|
48
64
|
version: "0"
|
49
|
-
type: :
|
50
|
-
version_requirements: *
|
65
|
+
type: :development
|
66
|
+
version_requirements: *id003
|
51
67
|
description: Bandit provides a way to do multi-armed bandit optimization of alternatives in a rails website
|
52
68
|
email:
|
53
69
|
- brian.muller@livingsocial.com
|
@@ -69,6 +85,7 @@ files:
|
|
69
85
|
- lib/bandit/date_hour.rb
|
70
86
|
- lib/bandit/exceptions.rb
|
71
87
|
- lib/bandit/experiment.rb
|
88
|
+
- lib/bandit/extensions/array.rb
|
72
89
|
- lib/bandit/extensions/controller_concerns.rb
|
73
90
|
- lib/bandit/extensions/string.rb
|
74
91
|
- lib/bandit/extensions/time.rb
|
@@ -111,6 +128,7 @@ files:
|
|
111
128
|
- test/memory_storage_test.rb
|
112
129
|
- test/redis_storage_test.rb
|
113
130
|
- test/storage_test_base.rb
|
131
|
+
- whybandit.rdoc
|
114
132
|
has_rdoc: true
|
115
133
|
homepage: https://github.com/bmuller/bandit
|
116
134
|
licenses: []
|