verdict 0.2.0 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -0
  3. data/Gemfile +4 -0
  4. data/README.md +56 -33
  5. data/lib/verdict.rb +2 -1
  6. data/lib/verdict/experiment.rb +15 -7
  7. data/lib/verdict/segmenters.rb +4 -0
  8. data/lib/verdict/segmenters/base_segmenter.rb +84 -0
  9. data/lib/verdict/segmenters/fixed_percentage_segmenter.rb +56 -0
  10. data/lib/verdict/segmenters/rollout_segmenter.rb +55 -0
  11. data/lib/verdict/segmenters/static_segmenter.rb +27 -0
  12. data/lib/verdict/storage.rb +3 -138
  13. data/lib/verdict/storage/memory_storage.rb +40 -0
  14. data/lib/verdict/storage/mock_storage.rb +38 -0
  15. data/lib/verdict/storage/redis_storage.rb +62 -0
  16. data/lib/verdict/tasks.rake +8 -8
  17. data/lib/verdict/version.rb +1 -1
  18. data/test/assignment_test.rb +4 -4
  19. data/test/conversion_test.rb +1 -1
  20. data/test/event_logger_test.rb +1 -1
  21. data/test/experiment_test.rb +12 -3
  22. data/test/experiments_repository_test.rb +1 -1
  23. data/test/group_test.rb +2 -2
  24. data/test/metadata_test.rb +1 -1
  25. data/test/{fixed_percentage_segmenter_test.rb → segmenters/fixed_percentage_segmenter_test.rb} +11 -11
  26. data/test/{rollout_segmenter_test.rb → segmenters/rollout_segmenter_test.rb} +2 -2
  27. data/test/{static_segmenter_test.rb → segmenters/static_segmenter_test.rb} +2 -2
  28. data/test/{memory_subject_storage_test.rb → storage/memory_subject_storage_test.rb} +1 -1
  29. data/test/{redis_subject_storage_test.rb → storage/redis_subject_storage_test.rb} +2 -2
  30. data/test/test_helper.rb +8 -0
  31. data/verdict.gemspec +6 -6
  32. metadata +24 -19
  33. data/lib/verdict/fixed_percentage_segmenter.rb +0 -52
  34. data/lib/verdict/rollout_segmenter.rb +0 -53
  35. data/lib/verdict/segmenter.rb +0 -87
  36. data/lib/verdict/static_segmenter.rb +0 -24
@@ -1,138 +1,3 @@
1
- require 'json'
2
-
3
- module Verdict::Storage
4
-
5
- class MockStorage
6
-
7
- # Should store the assignments to allow quick lookups.
8
- # - Assignments should be unique on the combination of
9
- # `assignment.experiment.handle` and `assignment.subject_identifier`.
10
- # - The main property to store is `group.handle`
11
- # - Should return true if stored successfully.
12
- def store_assignment(assignment)
13
- false
14
- end
15
-
16
- # Should do a fast lookup of an assignment of the subject for the given experiment.
17
- # - Should return nil if not found in store
18
- # - Should return an Assignment instance otherwise.
19
- def retrieve_assignment(experiment, subject_identifier)
20
- nil
21
- end
22
-
23
- # Should remove the subject from storage, so it will be reassigned later.
24
- def remove_assignment(experiment, subject_identifier)
25
- end
26
-
27
- # Should clear out the storage used for this experiment
28
- def clear_experiment(experiment)
29
- end
30
-
31
- # Retrieves the start timestamp of the experiment
32
- def retrieve_start_timestamp(experiment)
33
- nil
34
- end
35
-
36
- # Stores the timestamp on which the experiment was started
37
- def store_start_timestamp(experiment, timestamp)
38
- end
39
- end
40
-
41
- class MemoryStorage
42
-
43
- attr_reader :assignments, :start_timestamps
44
-
45
- def initialize
46
- @assignments = {}
47
- @start_timestamps = {}
48
- end
49
-
50
- def store_assignment(assignment)
51
- @assignments[assignment.experiment.handle] ||= {}
52
- @assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
53
- true
54
- end
55
-
56
- def retrieve_assignment(experiment, subject_identifier)
57
- experiment_store = @assignments[experiment.handle] || {}
58
- experiment_store[subject_identifier]
59
- end
60
-
61
- def remove_assignment(experiment, subject_identifier)
62
- @assignments[assignment.experiment.handle] ||= {}
63
- @assignments[assignment.experiment.handle].delete(subject_identifier)
64
- end
65
-
66
- def clear_experiment(experiment)
67
- @assignments.delete(experiment.handle)
68
- end
69
-
70
- def retrieve_start_timestamp(experiment)
71
- @start_timestamps[experiment.handle]
72
- end
73
-
74
- def store_start_timestamp(experiment, timestamp)
75
- @start_timestamps[experiment.handle] = timestamp
76
- end
77
- end
78
-
79
- class RedisStorage
80
-
81
- attr_accessor :redis, :key_prefix
82
-
83
- def initialize(redis = nil, options = {})
84
- @redis = redis
85
- @key_prefix = options[:key_prefix] || 'experiments/'
86
- end
87
-
88
- def retrieve_assignment(experiment, subject_identifier)
89
- if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
90
- hash = JSON.parse(value)
91
- experiment.subject_assignment(
92
- subject_identifier,
93
- experiment.group(hash['group']),
94
- DateTime.parse(hash['created_at']).to_time
95
- )
96
- end
97
- rescue ::Redis::BaseError => e
98
- raise Verdict::StorageError, "Redis error: #{e.message}"
99
- end
100
-
101
- def store_assignment(assignment)
102
- hash = { group: assignment.handle, created_at: assignment.created_at }
103
- redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
104
- rescue ::Redis::BaseError => e
105
- raise Verdict::StorageError, "Redis error: #{e.message}"
106
- end
107
-
108
- def remove_assignment(experiment, subject_identifier)
109
- redis.hdel(generate_experiment_key(experiment), subject_identifier)
110
- end
111
-
112
- def clear_experiment(experiment)
113
- redis.del(generate_experiment_key(experiment))
114
- redis.del(generate_experiment_start_timestamp_key(experiment))
115
- end
116
-
117
- def retrieve_start_timestamp(experiment)
118
- if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
119
- DateTime.parse(started_at).to_time
120
- end
121
- end
122
-
123
- def store_start_timestamp(experiment, timestamp)
124
- redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
125
- end
126
-
127
-
128
- private
129
-
130
- def generate_experiment_key(experiment)
131
- "#{@key_prefix}#{experiment.handle}"
132
- end
133
-
134
- def generate_experiment_start_timestamp_key(experiment)
135
- "#{@key_prefix}#{experiment.handle}/started_at"
136
- end
137
- end
138
- end
1
+ require 'verdict/storage/mock_storage'
2
+ require 'verdict/storage/memory_storage'
3
+ require 'verdict/storage/redis_storage'
@@ -0,0 +1,40 @@
1
+ module Verdict
2
+ module Storage
3
+ class MemoryStorage
4
+ attr_reader :assignments, :start_timestamps
5
+
6
+ def initialize
7
+ @assignments = {}
8
+ @start_timestamps = {}
9
+ end
10
+
11
+ def store_assignment(assignment)
12
+ @assignments[assignment.experiment.handle] ||= {}
13
+ @assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
14
+ true
15
+ end
16
+
17
+ def retrieve_assignment(experiment, subject_identifier)
18
+ experiment_store = @assignments[experiment.handle] || {}
19
+ experiment_store[subject_identifier]
20
+ end
21
+
22
+ def remove_assignment(experiment, subject_identifier)
23
+ @assignments[assignment.experiment.handle] ||= {}
24
+ @assignments[assignment.experiment.handle].delete(subject_identifier)
25
+ end
26
+
27
+ def clear_experiment(experiment)
28
+ @assignments.delete(experiment.handle)
29
+ end
30
+
31
+ def retrieve_start_timestamp(experiment)
32
+ @start_timestamps[experiment.handle]
33
+ end
34
+
35
+ def store_start_timestamp(experiment, timestamp)
36
+ @start_timestamps[experiment.handle] = timestamp
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,38 @@
1
+ module Verdict
2
+ module Storage
3
+ class MockStorage
4
+ # Should store the assignments to allow quick lookups.
5
+ # - Assignments should be unique on the combination of
6
+ # `assignment.experiment.handle` and `assignment.subject_identifier`.
7
+ # - The main property to store is `group.handle`
8
+ # - Should return true if stored successfully.
9
+ def store_assignment(assignment)
10
+ false
11
+ end
12
+
13
+ # Should do a fast lookup of an assignment of the subject for the given experiment.
14
+ # - Should return nil if not found in store
15
+ # - Should return an Assignment instance otherwise.
16
+ def retrieve_assignment(experiment, subject_identifier)
17
+ nil
18
+ end
19
+
20
+ # Should remove the subject from storage, so it will be reassigned later.
21
+ def remove_assignment(experiment, subject_identifier)
22
+ end
23
+
24
+ # Should clear out the storage used for this experiment
25
+ def clear_experiment(experiment)
26
+ end
27
+
28
+ # Retrieves the start timestamp of the experiment
29
+ def retrieve_start_timestamp(experiment)
30
+ nil
31
+ end
32
+
33
+ # Stores the timestamp on which the experiment was started
34
+ def store_start_timestamp(experiment, timestamp)
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,62 @@
1
+ module Verdict
2
+ module Storage
3
+ class RedisStorage
4
+ attr_accessor :redis, :key_prefix
5
+
6
+ def initialize(redis = nil, options = {})
7
+ @redis = redis
8
+ @key_prefix = options[:key_prefix] || 'experiments/'
9
+ end
10
+
11
+ def retrieve_assignment(experiment, subject_identifier)
12
+ if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
13
+ hash = JSON.parse(value)
14
+ experiment.subject_assignment(
15
+ subject_identifier,
16
+ experiment.group(hash['group']),
17
+ DateTime.parse(hash['created_at']).to_time
18
+ )
19
+ end
20
+ rescue ::Redis::BaseError => e
21
+ raise Verdict::StorageError, "Redis error: #{e.message}"
22
+ end
23
+
24
+ def store_assignment(assignment)
25
+ hash = { group: assignment.handle, created_at: assignment.created_at }
26
+ redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
27
+ rescue ::Redis::BaseError => e
28
+ raise Verdict::StorageError, "Redis error: #{e.message}"
29
+ end
30
+
31
+ def remove_assignment(experiment, subject_identifier)
32
+ redis.hdel(generate_experiment_key(experiment), subject_identifier)
33
+ end
34
+
35
+ def clear_experiment(experiment)
36
+ redis.del(generate_experiment_key(experiment))
37
+ redis.del(generate_experiment_start_timestamp_key(experiment))
38
+ end
39
+
40
+ def retrieve_start_timestamp(experiment)
41
+ if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
42
+ DateTime.parse(started_at).to_time
43
+ end
44
+ end
45
+
46
+ def store_start_timestamp(experiment, timestamp)
47
+ redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
48
+ end
49
+
50
+
51
+ private
52
+
53
+ def generate_experiment_key(experiment)
54
+ "#{@key_prefix}#{experiment.handle}"
55
+ end
56
+
57
+ def generate_experiment_start_timestamp_key(experiment)
58
+ "#{@key_prefix}#{experiment.handle}/started_at"
59
+ end
60
+ end
61
+ end
62
+ end
@@ -5,11 +5,11 @@ def require_env(key)
5
5
  end
6
6
 
7
7
  namespace :experiments do
8
-
8
+
9
9
  desc "List all defined experiments"
10
10
  task :list => 'environment' do
11
- length = Experiments.repository.keys.map(&:length).max
12
- Experiments.repository.each do |_, experiment|
11
+ length = Verdict.repository.keys.map(&:length).max
12
+ Verdict.repository.each do |_, experiment|
13
13
  print "#{experiment.handle.ljust(length)} | "
14
14
  print "Groups: #{experiment.groups.values.map(&:to_s).join(', ')}"
15
15
  puts
@@ -18,7 +18,7 @@ namespace :experiments do
18
18
 
19
19
  desc "Looks up the assignment for a given experiment and subject"
20
20
  task :lookup_assignment => 'environment' do
21
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
21
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
22
22
  subject_identifier = require_env('subject')
23
23
  assignment = experiment.lookup_assignment_for_identifier(subject_identifier)
24
24
  if assignment.nil?
@@ -32,7 +32,7 @@ namespace :experiments do
32
32
 
33
33
  desc "Manually assign a subject to a given group in an experiment"
34
34
  task :assign_manually => 'environment' do
35
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
35
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
36
36
  group = experiment.group(require_env('group')) or raise "Group not found"
37
37
  assignment = experiment.subject_assignment(require_env('subject'), group, false)
38
38
  experiment.store_assignment(assignment)
@@ -40,20 +40,20 @@ namespace :experiments do
40
40
 
41
41
  desc "Disqualify a subject from an experiment"
42
42
  task :disqualify => 'environment' do
43
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
43
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
44
44
  assignment = experiment.subject_assignment(require_env('subject'), nil, false)
45
45
  experiment.store_assignment(assignment)
46
46
  end
47
47
 
48
48
  desc "Removes the assignment for a subject so it will be reassigned to the experiment."
49
49
  task :remove_assignment => 'environment' do
50
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
50
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
51
51
  experiment.remove_subject_identifier(require_env('subject'))
52
52
  end
53
53
 
54
54
  desc "Runs the cleanup tasks for an experiment"
55
55
  task :wrapup => 'environment' do
56
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
56
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
57
57
  experiment.wrapup
58
58
  end
59
59
  end
@@ -1,3 +1,3 @@
1
1
  module Verdict
2
- VERSION = "0.2.0"
2
+ VERSION = "0.2.1"
3
3
  end
@@ -1,7 +1,7 @@
1
1
  require 'test_helper'
2
2
  require 'json'
3
3
 
4
- class AssignmentTest < MiniTest::Unit::TestCase
4
+ class AssignmentTest < Minitest::Test
5
5
 
6
6
  def setup
7
7
  @experiment = Verdict::Experiment.new('assignment test')
@@ -11,8 +11,8 @@ class AssignmentTest < MiniTest::Unit::TestCase
11
11
  def test_basic_properties
12
12
  assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
13
13
  assert_equal 'test_subject_id', assignment.subject_identifier
14
- assert_equal @experiment, assignment.experiment
15
- assert_equal @group, assignment.group
14
+ assert_equal @experiment, assignment.experiment
15
+ assert_equal @group, assignment.group
16
16
  assert assignment.returning?
17
17
  assert assignment.qualified?
18
18
  assert_equal :control, assignment.to_sym
@@ -20,7 +20,7 @@ class AssignmentTest < MiniTest::Unit::TestCase
20
20
  assert_kind_of Time, assignment.created_at
21
21
 
22
22
  non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
23
- assert_equal nil, non_assignment.group
23
+ assert_equal nil, non_assignment.group
24
24
  assert !non_assignment.returning?
25
25
  assert !non_assignment.qualified?
26
26
  assert_equal nil, non_assignment.to_sym
@@ -1,7 +1,7 @@
1
1
  require 'test_helper'
2
2
  require 'json'
3
3
 
4
- class ConversionTest < MiniTest::Unit::TestCase
4
+ class ConversionTest < Minitest::Test
5
5
 
6
6
  def setup
7
7
  @experiment = Verdict::Experiment.new('conversion test') do
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class EventLoggerTest < MiniTest::Unit::TestCase
3
+ class EventLoggerTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @experiment = Verdict::Experiment.new(:logger) do
@@ -1,7 +1,7 @@
1
1
  require 'json'
2
2
  require 'test_helper'
3
3
 
4
- class ExperimentTest < MiniTest::Unit::TestCase
4
+ class ExperimentTest < Minitest::Test
5
5
 
6
6
  def test_no_qualifier
7
7
  e = Verdict::Experiment.new('test')
@@ -109,7 +109,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
109
109
  mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
110
110
  mock_store.expects(:store_assignment).never
111
111
  e.assign(mock('subject'))
112
- end
112
+ end
113
113
 
114
114
  def test_new_unqualified_assignment_with_store_unqualified
115
115
  mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
@@ -202,7 +202,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
202
202
 
203
203
  conversion = e.convert(subject, :my_goal)
204
204
  assert_equal 'test_subject', conversion.subject_identifier
205
- assert_equal :my_goal, conversion.goal
205
+ assert_equal :my_goal, conversion.goal
206
206
  end
207
207
 
208
208
  def test_json
@@ -309,4 +309,13 @@ class ExperimentTest < MiniTest::Unit::TestCase
309
309
  e.assign(stub(id: '123'))
310
310
  assert e.started?, "The experiment should have started after the first assignment"
311
311
  end
312
+
313
+ def test_no_storage
314
+ e = Verdict::Experiment.new('starting_test') do
315
+ groups { group :all, 100 }
316
+ storage :none
317
+ end
318
+
319
+ assert_kind_of Verdict::Storage::MockStorage, e.storage
320
+ end
312
321
  end
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class ExperimentTest < MiniTest::Unit::TestCase
3
+ class ExperimentTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  Verdict.repository.clear
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class GroupTest < MiniTest::Unit::TestCase
3
+ class GroupTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @experiment = Verdict::Experiment.new('a')
@@ -38,5 +38,5 @@ class GroupTest < MiniTest::Unit::TestCase
38
38
  assert_equal 'control', json['handle']
39
39
  assert_equal 'testing', json['metadata']['name']
40
40
  assert_equal 'description', json['metadata']['description']
41
- end
41
+ end
42
42
  end