verdict 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/Gemfile +4 -0
- data/README.md +56 -33
- data/lib/verdict.rb +2 -1
- data/lib/verdict/experiment.rb +15 -7
- data/lib/verdict/segmenters.rb +4 -0
- data/lib/verdict/segmenters/base_segmenter.rb +84 -0
- data/lib/verdict/segmenters/fixed_percentage_segmenter.rb +56 -0
- data/lib/verdict/segmenters/rollout_segmenter.rb +55 -0
- data/lib/verdict/segmenters/static_segmenter.rb +27 -0
- data/lib/verdict/storage.rb +3 -138
- data/lib/verdict/storage/memory_storage.rb +40 -0
- data/lib/verdict/storage/mock_storage.rb +38 -0
- data/lib/verdict/storage/redis_storage.rb +62 -0
- data/lib/verdict/tasks.rake +8 -8
- data/lib/verdict/version.rb +1 -1
- data/test/assignment_test.rb +4 -4
- data/test/conversion_test.rb +1 -1
- data/test/event_logger_test.rb +1 -1
- data/test/experiment_test.rb +12 -3
- data/test/experiments_repository_test.rb +1 -1
- data/test/group_test.rb +2 -2
- data/test/metadata_test.rb +1 -1
- data/test/{fixed_percentage_segmenter_test.rb → segmenters/fixed_percentage_segmenter_test.rb} +11 -11
- data/test/{rollout_segmenter_test.rb → segmenters/rollout_segmenter_test.rb} +2 -2
- data/test/{static_segmenter_test.rb → segmenters/static_segmenter_test.rb} +2 -2
- data/test/{memory_subject_storage_test.rb → storage/memory_subject_storage_test.rb} +1 -1
- data/test/{redis_subject_storage_test.rb → storage/redis_subject_storage_test.rb} +2 -2
- data/test/test_helper.rb +8 -0
- data/verdict.gemspec +6 -6
- metadata +24 -19
- data/lib/verdict/fixed_percentage_segmenter.rb +0 -52
- data/lib/verdict/rollout_segmenter.rb +0 -53
- data/lib/verdict/segmenter.rb +0 -87
- data/lib/verdict/static_segmenter.rb +0 -24
data/lib/verdict/storage.rb
CHANGED
@@ -1,138 +1,3 @@
|
|
1
|
-
require '
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
class MockStorage
|
6
|
-
|
7
|
-
# Should store the assignments to allow quick lookups.
|
8
|
-
# - Assignments should be unique on the combination of
|
9
|
-
# `assignment.experiment.handle` and `assignment.subject_identifier`.
|
10
|
-
# - The main property to store is `group.handle`
|
11
|
-
# - Should return true if stored successfully.
|
12
|
-
def store_assignment(assignment)
|
13
|
-
false
|
14
|
-
end
|
15
|
-
|
16
|
-
# Should do a fast lookup of an assignment of the subject for the given experiment.
|
17
|
-
# - Should return nil if not found in store
|
18
|
-
# - Should return an Assignment instance otherwise.
|
19
|
-
def retrieve_assignment(experiment, subject_identifier)
|
20
|
-
nil
|
21
|
-
end
|
22
|
-
|
23
|
-
# Should remove the subject from storage, so it will be reassigned later.
|
24
|
-
def remove_assignment(experiment, subject_identifier)
|
25
|
-
end
|
26
|
-
|
27
|
-
# Should clear out the storage used for this experiment
|
28
|
-
def clear_experiment(experiment)
|
29
|
-
end
|
30
|
-
|
31
|
-
# Retrieves the start timestamp of the experiment
|
32
|
-
def retrieve_start_timestamp(experiment)
|
33
|
-
nil
|
34
|
-
end
|
35
|
-
|
36
|
-
# Stores the timestamp on which the experiment was started
|
37
|
-
def store_start_timestamp(experiment, timestamp)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
class MemoryStorage
|
42
|
-
|
43
|
-
attr_reader :assignments, :start_timestamps
|
44
|
-
|
45
|
-
def initialize
|
46
|
-
@assignments = {}
|
47
|
-
@start_timestamps = {}
|
48
|
-
end
|
49
|
-
|
50
|
-
def store_assignment(assignment)
|
51
|
-
@assignments[assignment.experiment.handle] ||= {}
|
52
|
-
@assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
|
53
|
-
true
|
54
|
-
end
|
55
|
-
|
56
|
-
def retrieve_assignment(experiment, subject_identifier)
|
57
|
-
experiment_store = @assignments[experiment.handle] || {}
|
58
|
-
experiment_store[subject_identifier]
|
59
|
-
end
|
60
|
-
|
61
|
-
def remove_assignment(experiment, subject_identifier)
|
62
|
-
@assignments[assignment.experiment.handle] ||= {}
|
63
|
-
@assignments[assignment.experiment.handle].delete(subject_identifier)
|
64
|
-
end
|
65
|
-
|
66
|
-
def clear_experiment(experiment)
|
67
|
-
@assignments.delete(experiment.handle)
|
68
|
-
end
|
69
|
-
|
70
|
-
def retrieve_start_timestamp(experiment)
|
71
|
-
@start_timestamps[experiment.handle]
|
72
|
-
end
|
73
|
-
|
74
|
-
def store_start_timestamp(experiment, timestamp)
|
75
|
-
@start_timestamps[experiment.handle] = timestamp
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
class RedisStorage
|
80
|
-
|
81
|
-
attr_accessor :redis, :key_prefix
|
82
|
-
|
83
|
-
def initialize(redis = nil, options = {})
|
84
|
-
@redis = redis
|
85
|
-
@key_prefix = options[:key_prefix] || 'experiments/'
|
86
|
-
end
|
87
|
-
|
88
|
-
def retrieve_assignment(experiment, subject_identifier)
|
89
|
-
if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
|
90
|
-
hash = JSON.parse(value)
|
91
|
-
experiment.subject_assignment(
|
92
|
-
subject_identifier,
|
93
|
-
experiment.group(hash['group']),
|
94
|
-
DateTime.parse(hash['created_at']).to_time
|
95
|
-
)
|
96
|
-
end
|
97
|
-
rescue ::Redis::BaseError => e
|
98
|
-
raise Verdict::StorageError, "Redis error: #{e.message}"
|
99
|
-
end
|
100
|
-
|
101
|
-
def store_assignment(assignment)
|
102
|
-
hash = { group: assignment.handle, created_at: assignment.created_at }
|
103
|
-
redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
|
104
|
-
rescue ::Redis::BaseError => e
|
105
|
-
raise Verdict::StorageError, "Redis error: #{e.message}"
|
106
|
-
end
|
107
|
-
|
108
|
-
def remove_assignment(experiment, subject_identifier)
|
109
|
-
redis.hdel(generate_experiment_key(experiment), subject_identifier)
|
110
|
-
end
|
111
|
-
|
112
|
-
def clear_experiment(experiment)
|
113
|
-
redis.del(generate_experiment_key(experiment))
|
114
|
-
redis.del(generate_experiment_start_timestamp_key(experiment))
|
115
|
-
end
|
116
|
-
|
117
|
-
def retrieve_start_timestamp(experiment)
|
118
|
-
if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
|
119
|
-
DateTime.parse(started_at).to_time
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
|
-
def store_start_timestamp(experiment, timestamp)
|
124
|
-
redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
|
125
|
-
end
|
126
|
-
|
127
|
-
|
128
|
-
private
|
129
|
-
|
130
|
-
def generate_experiment_key(experiment)
|
131
|
-
"#{@key_prefix}#{experiment.handle}"
|
132
|
-
end
|
133
|
-
|
134
|
-
def generate_experiment_start_timestamp_key(experiment)
|
135
|
-
"#{@key_prefix}#{experiment.handle}/started_at"
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
1
|
+
require 'verdict/storage/mock_storage'
|
2
|
+
require 'verdict/storage/memory_storage'
|
3
|
+
require 'verdict/storage/redis_storage'
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class MemoryStorage
|
4
|
+
attr_reader :assignments, :start_timestamps
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@assignments = {}
|
8
|
+
@start_timestamps = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def store_assignment(assignment)
|
12
|
+
@assignments[assignment.experiment.handle] ||= {}
|
13
|
+
@assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
def retrieve_assignment(experiment, subject_identifier)
|
18
|
+
experiment_store = @assignments[experiment.handle] || {}
|
19
|
+
experiment_store[subject_identifier]
|
20
|
+
end
|
21
|
+
|
22
|
+
def remove_assignment(experiment, subject_identifier)
|
23
|
+
@assignments[assignment.experiment.handle] ||= {}
|
24
|
+
@assignments[assignment.experiment.handle].delete(subject_identifier)
|
25
|
+
end
|
26
|
+
|
27
|
+
def clear_experiment(experiment)
|
28
|
+
@assignments.delete(experiment.handle)
|
29
|
+
end
|
30
|
+
|
31
|
+
def retrieve_start_timestamp(experiment)
|
32
|
+
@start_timestamps[experiment.handle]
|
33
|
+
end
|
34
|
+
|
35
|
+
def store_start_timestamp(experiment, timestamp)
|
36
|
+
@start_timestamps[experiment.handle] = timestamp
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class MockStorage
|
4
|
+
# Should store the assignments to allow quick lookups.
|
5
|
+
# - Assignments should be unique on the combination of
|
6
|
+
# `assignment.experiment.handle` and `assignment.subject_identifier`.
|
7
|
+
# - The main property to store is `group.handle`
|
8
|
+
# - Should return true if stored successfully.
|
9
|
+
def store_assignment(assignment)
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
# Should do a fast lookup of an assignment of the subject for the given experiment.
|
14
|
+
# - Should return nil if not found in store
|
15
|
+
# - Should return an Assignment instance otherwise.
|
16
|
+
def retrieve_assignment(experiment, subject_identifier)
|
17
|
+
nil
|
18
|
+
end
|
19
|
+
|
20
|
+
# Should remove the subject from storage, so it will be reassigned later.
|
21
|
+
def remove_assignment(experiment, subject_identifier)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Should clear out the storage used for this experiment
|
25
|
+
def clear_experiment(experiment)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Retrieves the start timestamp of the experiment
|
29
|
+
def retrieve_start_timestamp(experiment)
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
# Stores the timestamp on which the experiment was started
|
34
|
+
def store_start_timestamp(experiment, timestamp)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class RedisStorage
|
4
|
+
attr_accessor :redis, :key_prefix
|
5
|
+
|
6
|
+
def initialize(redis = nil, options = {})
|
7
|
+
@redis = redis
|
8
|
+
@key_prefix = options[:key_prefix] || 'experiments/'
|
9
|
+
end
|
10
|
+
|
11
|
+
def retrieve_assignment(experiment, subject_identifier)
|
12
|
+
if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
|
13
|
+
hash = JSON.parse(value)
|
14
|
+
experiment.subject_assignment(
|
15
|
+
subject_identifier,
|
16
|
+
experiment.group(hash['group']),
|
17
|
+
DateTime.parse(hash['created_at']).to_time
|
18
|
+
)
|
19
|
+
end
|
20
|
+
rescue ::Redis::BaseError => e
|
21
|
+
raise Verdict::StorageError, "Redis error: #{e.message}"
|
22
|
+
end
|
23
|
+
|
24
|
+
def store_assignment(assignment)
|
25
|
+
hash = { group: assignment.handle, created_at: assignment.created_at }
|
26
|
+
redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
|
27
|
+
rescue ::Redis::BaseError => e
|
28
|
+
raise Verdict::StorageError, "Redis error: #{e.message}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def remove_assignment(experiment, subject_identifier)
|
32
|
+
redis.hdel(generate_experiment_key(experiment), subject_identifier)
|
33
|
+
end
|
34
|
+
|
35
|
+
def clear_experiment(experiment)
|
36
|
+
redis.del(generate_experiment_key(experiment))
|
37
|
+
redis.del(generate_experiment_start_timestamp_key(experiment))
|
38
|
+
end
|
39
|
+
|
40
|
+
def retrieve_start_timestamp(experiment)
|
41
|
+
if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
|
42
|
+
DateTime.parse(started_at).to_time
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def store_start_timestamp(experiment, timestamp)
|
47
|
+
redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def generate_experiment_key(experiment)
|
54
|
+
"#{@key_prefix}#{experiment.handle}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def generate_experiment_start_timestamp_key(experiment)
|
58
|
+
"#{@key_prefix}#{experiment.handle}/started_at"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/verdict/tasks.rake
CHANGED
@@ -5,11 +5,11 @@ def require_env(key)
|
|
5
5
|
end
|
6
6
|
|
7
7
|
namespace :experiments do
|
8
|
-
|
8
|
+
|
9
9
|
desc "List all defined experiments"
|
10
10
|
task :list => 'environment' do
|
11
|
-
length =
|
12
|
-
|
11
|
+
length = Verdict.repository.keys.map(&:length).max
|
12
|
+
Verdict.repository.each do |_, experiment|
|
13
13
|
print "#{experiment.handle.ljust(length)} | "
|
14
14
|
print "Groups: #{experiment.groups.values.map(&:to_s).join(', ')}"
|
15
15
|
puts
|
@@ -18,7 +18,7 @@ namespace :experiments do
|
|
18
18
|
|
19
19
|
desc "Looks up the assignment for a given experiment and subject"
|
20
20
|
task :lookup_assignment => 'environment' do
|
21
|
-
experiment =
|
21
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
22
22
|
subject_identifier = require_env('subject')
|
23
23
|
assignment = experiment.lookup_assignment_for_identifier(subject_identifier)
|
24
24
|
if assignment.nil?
|
@@ -32,7 +32,7 @@ namespace :experiments do
|
|
32
32
|
|
33
33
|
desc "Manually assign a subject to a given group in an experiment"
|
34
34
|
task :assign_manually => 'environment' do
|
35
|
-
experiment =
|
35
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
36
36
|
group = experiment.group(require_env('group')) or raise "Group not found"
|
37
37
|
assignment = experiment.subject_assignment(require_env('subject'), group, false)
|
38
38
|
experiment.store_assignment(assignment)
|
@@ -40,20 +40,20 @@ namespace :experiments do
|
|
40
40
|
|
41
41
|
desc "Disqualify a subject from an experiment"
|
42
42
|
task :disqualify => 'environment' do
|
43
|
-
experiment =
|
43
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
44
44
|
assignment = experiment.subject_assignment(require_env('subject'), nil, false)
|
45
45
|
experiment.store_assignment(assignment)
|
46
46
|
end
|
47
47
|
|
48
48
|
desc "Removes the assignment for a subject so it will be reassigned to the experiment."
|
49
49
|
task :remove_assignment => 'environment' do
|
50
|
-
experiment =
|
50
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
51
51
|
experiment.remove_subject_identifier(require_env('subject'))
|
52
52
|
end
|
53
53
|
|
54
54
|
desc "Runs the cleanup tasks for an experiment"
|
55
55
|
task :wrapup => 'environment' do
|
56
|
-
experiment =
|
56
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
57
57
|
experiment.wrapup
|
58
58
|
end
|
59
59
|
end
|
data/lib/verdict/version.rb
CHANGED
data/test/assignment_test.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
require 'json'
|
3
3
|
|
4
|
-
class AssignmentTest <
|
4
|
+
class AssignmentTest < Minitest::Test
|
5
5
|
|
6
6
|
def setup
|
7
7
|
@experiment = Verdict::Experiment.new('assignment test')
|
@@ -11,8 +11,8 @@ class AssignmentTest < MiniTest::Unit::TestCase
|
|
11
11
|
def test_basic_properties
|
12
12
|
assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
|
13
13
|
assert_equal 'test_subject_id', assignment.subject_identifier
|
14
|
-
assert_equal @experiment, assignment.experiment
|
15
|
-
assert_equal @group, assignment.group
|
14
|
+
assert_equal @experiment, assignment.experiment
|
15
|
+
assert_equal @group, assignment.group
|
16
16
|
assert assignment.returning?
|
17
17
|
assert assignment.qualified?
|
18
18
|
assert_equal :control, assignment.to_sym
|
@@ -20,7 +20,7 @@ class AssignmentTest < MiniTest::Unit::TestCase
|
|
20
20
|
assert_kind_of Time, assignment.created_at
|
21
21
|
|
22
22
|
non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
|
23
|
-
assert_equal nil, non_assignment.group
|
23
|
+
assert_equal nil, non_assignment.group
|
24
24
|
assert !non_assignment.returning?
|
25
25
|
assert !non_assignment.qualified?
|
26
26
|
assert_equal nil, non_assignment.to_sym
|
data/test/conversion_test.rb
CHANGED
data/test/event_logger_test.rb
CHANGED
data/test/experiment_test.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'test_helper'
|
3
3
|
|
4
|
-
class ExperimentTest <
|
4
|
+
class ExperimentTest < Minitest::Test
|
5
5
|
|
6
6
|
def test_no_qualifier
|
7
7
|
e = Verdict::Experiment.new('test')
|
@@ -109,7 +109,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
109
109
|
mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
|
110
110
|
mock_store.expects(:store_assignment).never
|
111
111
|
e.assign(mock('subject'))
|
112
|
-
end
|
112
|
+
end
|
113
113
|
|
114
114
|
def test_new_unqualified_assignment_with_store_unqualified
|
115
115
|
mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
|
@@ -202,7 +202,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
202
202
|
|
203
203
|
conversion = e.convert(subject, :my_goal)
|
204
204
|
assert_equal 'test_subject', conversion.subject_identifier
|
205
|
-
assert_equal :my_goal, conversion.goal
|
205
|
+
assert_equal :my_goal, conversion.goal
|
206
206
|
end
|
207
207
|
|
208
208
|
def test_json
|
@@ -309,4 +309,13 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
309
309
|
e.assign(stub(id: '123'))
|
310
310
|
assert e.started?, "The experiment should have started after the first assignment"
|
311
311
|
end
|
312
|
+
|
313
|
+
def test_no_storage
|
314
|
+
e = Verdict::Experiment.new('starting_test') do
|
315
|
+
groups { group :all, 100 }
|
316
|
+
storage :none
|
317
|
+
end
|
318
|
+
|
319
|
+
assert_kind_of Verdict::Storage::MockStorage, e.storage
|
320
|
+
end
|
312
321
|
end
|
data/test/group_test.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
|
-
class GroupTest <
|
3
|
+
class GroupTest < Minitest::Test
|
4
4
|
|
5
5
|
def setup
|
6
6
|
@experiment = Verdict::Experiment.new('a')
|
@@ -38,5 +38,5 @@ class GroupTest < MiniTest::Unit::TestCase
|
|
38
38
|
assert_equal 'control', json['handle']
|
39
39
|
assert_equal 'testing', json['metadata']['name']
|
40
40
|
assert_equal 'description', json['metadata']['description']
|
41
|
-
end
|
41
|
+
end
|
42
42
|
end
|