verdict 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/Gemfile +4 -0
- data/README.md +56 -33
- data/lib/verdict.rb +2 -1
- data/lib/verdict/experiment.rb +15 -7
- data/lib/verdict/segmenters.rb +4 -0
- data/lib/verdict/segmenters/base_segmenter.rb +84 -0
- data/lib/verdict/segmenters/fixed_percentage_segmenter.rb +56 -0
- data/lib/verdict/segmenters/rollout_segmenter.rb +55 -0
- data/lib/verdict/segmenters/static_segmenter.rb +27 -0
- data/lib/verdict/storage.rb +3 -138
- data/lib/verdict/storage/memory_storage.rb +40 -0
- data/lib/verdict/storage/mock_storage.rb +38 -0
- data/lib/verdict/storage/redis_storage.rb +62 -0
- data/lib/verdict/tasks.rake +8 -8
- data/lib/verdict/version.rb +1 -1
- data/test/assignment_test.rb +4 -4
- data/test/conversion_test.rb +1 -1
- data/test/event_logger_test.rb +1 -1
- data/test/experiment_test.rb +12 -3
- data/test/experiments_repository_test.rb +1 -1
- data/test/group_test.rb +2 -2
- data/test/metadata_test.rb +1 -1
- data/test/{fixed_percentage_segmenter_test.rb → segmenters/fixed_percentage_segmenter_test.rb} +11 -11
- data/test/{rollout_segmenter_test.rb → segmenters/rollout_segmenter_test.rb} +2 -2
- data/test/{static_segmenter_test.rb → segmenters/static_segmenter_test.rb} +2 -2
- data/test/{memory_subject_storage_test.rb → storage/memory_subject_storage_test.rb} +1 -1
- data/test/{redis_subject_storage_test.rb → storage/redis_subject_storage_test.rb} +2 -2
- data/test/test_helper.rb +8 -0
- data/verdict.gemspec +6 -6
- metadata +24 -19
- data/lib/verdict/fixed_percentage_segmenter.rb +0 -52
- data/lib/verdict/rollout_segmenter.rb +0 -53
- data/lib/verdict/segmenter.rb +0 -87
- data/lib/verdict/static_segmenter.rb +0 -24
data/lib/verdict/storage.rb
CHANGED
@@ -1,138 +1,3 @@
|
|
1
|
-
require '
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
class MockStorage
|
6
|
-
|
7
|
-
# Should store the assignments to allow quick lookups.
|
8
|
-
# - Assignments should be unique on the combination of
|
9
|
-
# `assignment.experiment.handle` and `assignment.subject_identifier`.
|
10
|
-
# - The main property to store is `group.handle`
|
11
|
-
# - Should return true if stored successfully.
|
12
|
-
def store_assignment(assignment)
|
13
|
-
false
|
14
|
-
end
|
15
|
-
|
16
|
-
# Should do a fast lookup of an assignment of the subject for the given experiment.
|
17
|
-
# - Should return nil if not found in store
|
18
|
-
# - Should return an Assignment instance otherwise.
|
19
|
-
def retrieve_assignment(experiment, subject_identifier)
|
20
|
-
nil
|
21
|
-
end
|
22
|
-
|
23
|
-
# Should remove the subject from storage, so it will be reassigned later.
|
24
|
-
def remove_assignment(experiment, subject_identifier)
|
25
|
-
end
|
26
|
-
|
27
|
-
# Should clear out the storage used for this experiment
|
28
|
-
def clear_experiment(experiment)
|
29
|
-
end
|
30
|
-
|
31
|
-
# Retrieves the start timestamp of the experiment
|
32
|
-
def retrieve_start_timestamp(experiment)
|
33
|
-
nil
|
34
|
-
end
|
35
|
-
|
36
|
-
# Stores the timestamp on which the experiment was started
|
37
|
-
def store_start_timestamp(experiment, timestamp)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
class MemoryStorage
|
42
|
-
|
43
|
-
attr_reader :assignments, :start_timestamps
|
44
|
-
|
45
|
-
def initialize
|
46
|
-
@assignments = {}
|
47
|
-
@start_timestamps = {}
|
48
|
-
end
|
49
|
-
|
50
|
-
def store_assignment(assignment)
|
51
|
-
@assignments[assignment.experiment.handle] ||= {}
|
52
|
-
@assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
|
53
|
-
true
|
54
|
-
end
|
55
|
-
|
56
|
-
def retrieve_assignment(experiment, subject_identifier)
|
57
|
-
experiment_store = @assignments[experiment.handle] || {}
|
58
|
-
experiment_store[subject_identifier]
|
59
|
-
end
|
60
|
-
|
61
|
-
def remove_assignment(experiment, subject_identifier)
|
62
|
-
@assignments[assignment.experiment.handle] ||= {}
|
63
|
-
@assignments[assignment.experiment.handle].delete(subject_identifier)
|
64
|
-
end
|
65
|
-
|
66
|
-
def clear_experiment(experiment)
|
67
|
-
@assignments.delete(experiment.handle)
|
68
|
-
end
|
69
|
-
|
70
|
-
def retrieve_start_timestamp(experiment)
|
71
|
-
@start_timestamps[experiment.handle]
|
72
|
-
end
|
73
|
-
|
74
|
-
def store_start_timestamp(experiment, timestamp)
|
75
|
-
@start_timestamps[experiment.handle] = timestamp
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
class RedisStorage
|
80
|
-
|
81
|
-
attr_accessor :redis, :key_prefix
|
82
|
-
|
83
|
-
def initialize(redis = nil, options = {})
|
84
|
-
@redis = redis
|
85
|
-
@key_prefix = options[:key_prefix] || 'experiments/'
|
86
|
-
end
|
87
|
-
|
88
|
-
def retrieve_assignment(experiment, subject_identifier)
|
89
|
-
if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
|
90
|
-
hash = JSON.parse(value)
|
91
|
-
experiment.subject_assignment(
|
92
|
-
subject_identifier,
|
93
|
-
experiment.group(hash['group']),
|
94
|
-
DateTime.parse(hash['created_at']).to_time
|
95
|
-
)
|
96
|
-
end
|
97
|
-
rescue ::Redis::BaseError => e
|
98
|
-
raise Verdict::StorageError, "Redis error: #{e.message}"
|
99
|
-
end
|
100
|
-
|
101
|
-
def store_assignment(assignment)
|
102
|
-
hash = { group: assignment.handle, created_at: assignment.created_at }
|
103
|
-
redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
|
104
|
-
rescue ::Redis::BaseError => e
|
105
|
-
raise Verdict::StorageError, "Redis error: #{e.message}"
|
106
|
-
end
|
107
|
-
|
108
|
-
def remove_assignment(experiment, subject_identifier)
|
109
|
-
redis.hdel(generate_experiment_key(experiment), subject_identifier)
|
110
|
-
end
|
111
|
-
|
112
|
-
def clear_experiment(experiment)
|
113
|
-
redis.del(generate_experiment_key(experiment))
|
114
|
-
redis.del(generate_experiment_start_timestamp_key(experiment))
|
115
|
-
end
|
116
|
-
|
117
|
-
def retrieve_start_timestamp(experiment)
|
118
|
-
if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
|
119
|
-
DateTime.parse(started_at).to_time
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
|
-
def store_start_timestamp(experiment, timestamp)
|
124
|
-
redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
|
125
|
-
end
|
126
|
-
|
127
|
-
|
128
|
-
private
|
129
|
-
|
130
|
-
def generate_experiment_key(experiment)
|
131
|
-
"#{@key_prefix}#{experiment.handle}"
|
132
|
-
end
|
133
|
-
|
134
|
-
def generate_experiment_start_timestamp_key(experiment)
|
135
|
-
"#{@key_prefix}#{experiment.handle}/started_at"
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
1
|
+
require 'verdict/storage/mock_storage'
|
2
|
+
require 'verdict/storage/memory_storage'
|
3
|
+
require 'verdict/storage/redis_storage'
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class MemoryStorage
|
4
|
+
attr_reader :assignments, :start_timestamps
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@assignments = {}
|
8
|
+
@start_timestamps = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def store_assignment(assignment)
|
12
|
+
@assignments[assignment.experiment.handle] ||= {}
|
13
|
+
@assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
def retrieve_assignment(experiment, subject_identifier)
|
18
|
+
experiment_store = @assignments[experiment.handle] || {}
|
19
|
+
experiment_store[subject_identifier]
|
20
|
+
end
|
21
|
+
|
22
|
+
def remove_assignment(experiment, subject_identifier)
|
23
|
+
@assignments[assignment.experiment.handle] ||= {}
|
24
|
+
@assignments[assignment.experiment.handle].delete(subject_identifier)
|
25
|
+
end
|
26
|
+
|
27
|
+
def clear_experiment(experiment)
|
28
|
+
@assignments.delete(experiment.handle)
|
29
|
+
end
|
30
|
+
|
31
|
+
def retrieve_start_timestamp(experiment)
|
32
|
+
@start_timestamps[experiment.handle]
|
33
|
+
end
|
34
|
+
|
35
|
+
def store_start_timestamp(experiment, timestamp)
|
36
|
+
@start_timestamps[experiment.handle] = timestamp
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class MockStorage
|
4
|
+
# Should store the assignments to allow quick lookups.
|
5
|
+
# - Assignments should be unique on the combination of
|
6
|
+
# `assignment.experiment.handle` and `assignment.subject_identifier`.
|
7
|
+
# - The main property to store is `group.handle`
|
8
|
+
# - Should return true if stored successfully.
|
9
|
+
def store_assignment(assignment)
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
# Should do a fast lookup of an assignment of the subject for the given experiment.
|
14
|
+
# - Should return nil if not found in store
|
15
|
+
# - Should return an Assignment instance otherwise.
|
16
|
+
def retrieve_assignment(experiment, subject_identifier)
|
17
|
+
nil
|
18
|
+
end
|
19
|
+
|
20
|
+
# Should remove the subject from storage, so it will be reassigned later.
|
21
|
+
def remove_assignment(experiment, subject_identifier)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Should clear out the storage used for this experiment
|
25
|
+
def clear_experiment(experiment)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Retrieves the start timestamp of the experiment
|
29
|
+
def retrieve_start_timestamp(experiment)
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
# Stores the timestamp on which the experiment was started
|
34
|
+
def store_start_timestamp(experiment, timestamp)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Verdict
|
2
|
+
module Storage
|
3
|
+
class RedisStorage
|
4
|
+
attr_accessor :redis, :key_prefix
|
5
|
+
|
6
|
+
def initialize(redis = nil, options = {})
|
7
|
+
@redis = redis
|
8
|
+
@key_prefix = options[:key_prefix] || 'experiments/'
|
9
|
+
end
|
10
|
+
|
11
|
+
def retrieve_assignment(experiment, subject_identifier)
|
12
|
+
if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
|
13
|
+
hash = JSON.parse(value)
|
14
|
+
experiment.subject_assignment(
|
15
|
+
subject_identifier,
|
16
|
+
experiment.group(hash['group']),
|
17
|
+
DateTime.parse(hash['created_at']).to_time
|
18
|
+
)
|
19
|
+
end
|
20
|
+
rescue ::Redis::BaseError => e
|
21
|
+
raise Verdict::StorageError, "Redis error: #{e.message}"
|
22
|
+
end
|
23
|
+
|
24
|
+
def store_assignment(assignment)
|
25
|
+
hash = { group: assignment.handle, created_at: assignment.created_at }
|
26
|
+
redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
|
27
|
+
rescue ::Redis::BaseError => e
|
28
|
+
raise Verdict::StorageError, "Redis error: #{e.message}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def remove_assignment(experiment, subject_identifier)
|
32
|
+
redis.hdel(generate_experiment_key(experiment), subject_identifier)
|
33
|
+
end
|
34
|
+
|
35
|
+
def clear_experiment(experiment)
|
36
|
+
redis.del(generate_experiment_key(experiment))
|
37
|
+
redis.del(generate_experiment_start_timestamp_key(experiment))
|
38
|
+
end
|
39
|
+
|
40
|
+
def retrieve_start_timestamp(experiment)
|
41
|
+
if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
|
42
|
+
DateTime.parse(started_at).to_time
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def store_start_timestamp(experiment, timestamp)
|
47
|
+
redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def generate_experiment_key(experiment)
|
54
|
+
"#{@key_prefix}#{experiment.handle}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def generate_experiment_start_timestamp_key(experiment)
|
58
|
+
"#{@key_prefix}#{experiment.handle}/started_at"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/verdict/tasks.rake
CHANGED
@@ -5,11 +5,11 @@ def require_env(key)
|
|
5
5
|
end
|
6
6
|
|
7
7
|
namespace :experiments do
|
8
|
-
|
8
|
+
|
9
9
|
desc "List all defined experiments"
|
10
10
|
task :list => 'environment' do
|
11
|
-
length =
|
12
|
-
|
11
|
+
length = Verdict.repository.keys.map(&:length).max
|
12
|
+
Verdict.repository.each do |_, experiment|
|
13
13
|
print "#{experiment.handle.ljust(length)} | "
|
14
14
|
print "Groups: #{experiment.groups.values.map(&:to_s).join(', ')}"
|
15
15
|
puts
|
@@ -18,7 +18,7 @@ namespace :experiments do
|
|
18
18
|
|
19
19
|
desc "Looks up the assignment for a given experiment and subject"
|
20
20
|
task :lookup_assignment => 'environment' do
|
21
|
-
experiment =
|
21
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
22
22
|
subject_identifier = require_env('subject')
|
23
23
|
assignment = experiment.lookup_assignment_for_identifier(subject_identifier)
|
24
24
|
if assignment.nil?
|
@@ -32,7 +32,7 @@ namespace :experiments do
|
|
32
32
|
|
33
33
|
desc "Manually assign a subject to a given group in an experiment"
|
34
34
|
task :assign_manually => 'environment' do
|
35
|
-
experiment =
|
35
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
36
36
|
group = experiment.group(require_env('group')) or raise "Group not found"
|
37
37
|
assignment = experiment.subject_assignment(require_env('subject'), group, false)
|
38
38
|
experiment.store_assignment(assignment)
|
@@ -40,20 +40,20 @@ namespace :experiments do
|
|
40
40
|
|
41
41
|
desc "Disqualify a subject from an experiment"
|
42
42
|
task :disqualify => 'environment' do
|
43
|
-
experiment =
|
43
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
44
44
|
assignment = experiment.subject_assignment(require_env('subject'), nil, false)
|
45
45
|
experiment.store_assignment(assignment)
|
46
46
|
end
|
47
47
|
|
48
48
|
desc "Removes the assignment for a subject so it will be reassigned to the experiment."
|
49
49
|
task :remove_assignment => 'environment' do
|
50
|
-
experiment =
|
50
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
51
51
|
experiment.remove_subject_identifier(require_env('subject'))
|
52
52
|
end
|
53
53
|
|
54
54
|
desc "Runs the cleanup tasks for an experiment"
|
55
55
|
task :wrapup => 'environment' do
|
56
|
-
experiment =
|
56
|
+
experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
|
57
57
|
experiment.wrapup
|
58
58
|
end
|
59
59
|
end
|
data/lib/verdict/version.rb
CHANGED
data/test/assignment_test.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
require 'json'
|
3
3
|
|
4
|
-
class AssignmentTest <
|
4
|
+
class AssignmentTest < Minitest::Test
|
5
5
|
|
6
6
|
def setup
|
7
7
|
@experiment = Verdict::Experiment.new('assignment test')
|
@@ -11,8 +11,8 @@ class AssignmentTest < MiniTest::Unit::TestCase
|
|
11
11
|
def test_basic_properties
|
12
12
|
assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
|
13
13
|
assert_equal 'test_subject_id', assignment.subject_identifier
|
14
|
-
assert_equal @experiment, assignment.experiment
|
15
|
-
assert_equal @group, assignment.group
|
14
|
+
assert_equal @experiment, assignment.experiment
|
15
|
+
assert_equal @group, assignment.group
|
16
16
|
assert assignment.returning?
|
17
17
|
assert assignment.qualified?
|
18
18
|
assert_equal :control, assignment.to_sym
|
@@ -20,7 +20,7 @@ class AssignmentTest < MiniTest::Unit::TestCase
|
|
20
20
|
assert_kind_of Time, assignment.created_at
|
21
21
|
|
22
22
|
non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
|
23
|
-
assert_equal nil, non_assignment.group
|
23
|
+
assert_equal nil, non_assignment.group
|
24
24
|
assert !non_assignment.returning?
|
25
25
|
assert !non_assignment.qualified?
|
26
26
|
assert_equal nil, non_assignment.to_sym
|
data/test/conversion_test.rb
CHANGED
data/test/event_logger_test.rb
CHANGED
data/test/experiment_test.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'test_helper'
|
3
3
|
|
4
|
-
class ExperimentTest <
|
4
|
+
class ExperimentTest < Minitest::Test
|
5
5
|
|
6
6
|
def test_no_qualifier
|
7
7
|
e = Verdict::Experiment.new('test')
|
@@ -109,7 +109,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
109
109
|
mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
|
110
110
|
mock_store.expects(:store_assignment).never
|
111
111
|
e.assign(mock('subject'))
|
112
|
-
end
|
112
|
+
end
|
113
113
|
|
114
114
|
def test_new_unqualified_assignment_with_store_unqualified
|
115
115
|
mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
|
@@ -202,7 +202,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
202
202
|
|
203
203
|
conversion = e.convert(subject, :my_goal)
|
204
204
|
assert_equal 'test_subject', conversion.subject_identifier
|
205
|
-
assert_equal :my_goal, conversion.goal
|
205
|
+
assert_equal :my_goal, conversion.goal
|
206
206
|
end
|
207
207
|
|
208
208
|
def test_json
|
@@ -309,4 +309,13 @@ class ExperimentTest < MiniTest::Unit::TestCase
|
|
309
309
|
e.assign(stub(id: '123'))
|
310
310
|
assert e.started?, "The experiment should have started after the first assignment"
|
311
311
|
end
|
312
|
+
|
313
|
+
def test_no_storage
|
314
|
+
e = Verdict::Experiment.new('starting_test') do
|
315
|
+
groups { group :all, 100 }
|
316
|
+
storage :none
|
317
|
+
end
|
318
|
+
|
319
|
+
assert_kind_of Verdict::Storage::MockStorage, e.storage
|
320
|
+
end
|
312
321
|
end
|
data/test/group_test.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
|
-
class GroupTest <
|
3
|
+
class GroupTest < Minitest::Test
|
4
4
|
|
5
5
|
def setup
|
6
6
|
@experiment = Verdict::Experiment.new('a')
|
@@ -38,5 +38,5 @@ class GroupTest < MiniTest::Unit::TestCase
|
|
38
38
|
assert_equal 'control', json['handle']
|
39
39
|
assert_equal 'testing', json['metadata']['name']
|
40
40
|
assert_equal 'description', json['metadata']['description']
|
41
|
-
end
|
41
|
+
end
|
42
42
|
end
|