verdict 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -0
  3. data/Gemfile +4 -0
  4. data/README.md +56 -33
  5. data/lib/verdict.rb +2 -1
  6. data/lib/verdict/experiment.rb +15 -7
  7. data/lib/verdict/segmenters.rb +4 -0
  8. data/lib/verdict/segmenters/base_segmenter.rb +84 -0
  9. data/lib/verdict/segmenters/fixed_percentage_segmenter.rb +56 -0
  10. data/lib/verdict/segmenters/rollout_segmenter.rb +55 -0
  11. data/lib/verdict/segmenters/static_segmenter.rb +27 -0
  12. data/lib/verdict/storage.rb +3 -138
  13. data/lib/verdict/storage/memory_storage.rb +40 -0
  14. data/lib/verdict/storage/mock_storage.rb +38 -0
  15. data/lib/verdict/storage/redis_storage.rb +62 -0
  16. data/lib/verdict/tasks.rake +8 -8
  17. data/lib/verdict/version.rb +1 -1
  18. data/test/assignment_test.rb +4 -4
  19. data/test/conversion_test.rb +1 -1
  20. data/test/event_logger_test.rb +1 -1
  21. data/test/experiment_test.rb +12 -3
  22. data/test/experiments_repository_test.rb +1 -1
  23. data/test/group_test.rb +2 -2
  24. data/test/metadata_test.rb +1 -1
  25. data/test/{fixed_percentage_segmenter_test.rb → segmenters/fixed_percentage_segmenter_test.rb} +11 -11
  26. data/test/{rollout_segmenter_test.rb → segmenters/rollout_segmenter_test.rb} +2 -2
  27. data/test/{static_segmenter_test.rb → segmenters/static_segmenter_test.rb} +2 -2
  28. data/test/{memory_subject_storage_test.rb → storage/memory_subject_storage_test.rb} +1 -1
  29. data/test/{redis_subject_storage_test.rb → storage/redis_subject_storage_test.rb} +2 -2
  30. data/test/test_helper.rb +8 -0
  31. data/verdict.gemspec +6 -6
  32. metadata +24 -19
  33. data/lib/verdict/fixed_percentage_segmenter.rb +0 -52
  34. data/lib/verdict/rollout_segmenter.rb +0 -53
  35. data/lib/verdict/segmenter.rb +0 -87
  36. data/lib/verdict/static_segmenter.rb +0 -24
@@ -1,138 +1,3 @@
1
- require 'json'
2
-
3
- module Verdict::Storage
4
-
5
- class MockStorage
6
-
7
- # Should store the assignments to allow quick lookups.
8
- # - Assignments should be unique on the combination of
9
- # `assignment.experiment.handle` and `assignment.subject_identifier`.
10
- # - The main property to store is `group.handle`
11
- # - Should return true if stored successfully.
12
- def store_assignment(assignment)
13
- false
14
- end
15
-
16
- # Should do a fast lookup of an assignment of the subject for the given experiment.
17
- # - Should return nil if not found in store
18
- # - Should return an Assignment instance otherwise.
19
- def retrieve_assignment(experiment, subject_identifier)
20
- nil
21
- end
22
-
23
- # Should remove the subject from storage, so it will be reassigned later.
24
- def remove_assignment(experiment, subject_identifier)
25
- end
26
-
27
- # Should clear out the storage used for this experiment
28
- def clear_experiment(experiment)
29
- end
30
-
31
- # Retrieves the start timestamp of the experiment
32
- def retrieve_start_timestamp(experiment)
33
- nil
34
- end
35
-
36
- # Stores the timestamp on which the experiment was started
37
- def store_start_timestamp(experiment, timestamp)
38
- end
39
- end
40
-
41
- class MemoryStorage
42
-
43
- attr_reader :assignments, :start_timestamps
44
-
45
- def initialize
46
- @assignments = {}
47
- @start_timestamps = {}
48
- end
49
-
50
- def store_assignment(assignment)
51
- @assignments[assignment.experiment.handle] ||= {}
52
- @assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
53
- true
54
- end
55
-
56
- def retrieve_assignment(experiment, subject_identifier)
57
- experiment_store = @assignments[experiment.handle] || {}
58
- experiment_store[subject_identifier]
59
- end
60
-
61
- def remove_assignment(experiment, subject_identifier)
62
- @assignments[assignment.experiment.handle] ||= {}
63
- @assignments[assignment.experiment.handle].delete(subject_identifier)
64
- end
65
-
66
- def clear_experiment(experiment)
67
- @assignments.delete(experiment.handle)
68
- end
69
-
70
- def retrieve_start_timestamp(experiment)
71
- @start_timestamps[experiment.handle]
72
- end
73
-
74
- def store_start_timestamp(experiment, timestamp)
75
- @start_timestamps[experiment.handle] = timestamp
76
- end
77
- end
78
-
79
- class RedisStorage
80
-
81
- attr_accessor :redis, :key_prefix
82
-
83
- def initialize(redis = nil, options = {})
84
- @redis = redis
85
- @key_prefix = options[:key_prefix] || 'experiments/'
86
- end
87
-
88
- def retrieve_assignment(experiment, subject_identifier)
89
- if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
90
- hash = JSON.parse(value)
91
- experiment.subject_assignment(
92
- subject_identifier,
93
- experiment.group(hash['group']),
94
- DateTime.parse(hash['created_at']).to_time
95
- )
96
- end
97
- rescue ::Redis::BaseError => e
98
- raise Verdict::StorageError, "Redis error: #{e.message}"
99
- end
100
-
101
- def store_assignment(assignment)
102
- hash = { group: assignment.handle, created_at: assignment.created_at }
103
- redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
104
- rescue ::Redis::BaseError => e
105
- raise Verdict::StorageError, "Redis error: #{e.message}"
106
- end
107
-
108
- def remove_assignment(experiment, subject_identifier)
109
- redis.hdel(generate_experiment_key(experiment), subject_identifier)
110
- end
111
-
112
- def clear_experiment(experiment)
113
- redis.del(generate_experiment_key(experiment))
114
- redis.del(generate_experiment_start_timestamp_key(experiment))
115
- end
116
-
117
- def retrieve_start_timestamp(experiment)
118
- if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
119
- DateTime.parse(started_at).to_time
120
- end
121
- end
122
-
123
- def store_start_timestamp(experiment, timestamp)
124
- redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
125
- end
126
-
127
-
128
- private
129
-
130
- def generate_experiment_key(experiment)
131
- "#{@key_prefix}#{experiment.handle}"
132
- end
133
-
134
- def generate_experiment_start_timestamp_key(experiment)
135
- "#{@key_prefix}#{experiment.handle}/started_at"
136
- end
137
- end
138
- end
1
+ require 'verdict/storage/mock_storage'
2
+ require 'verdict/storage/memory_storage'
3
+ require 'verdict/storage/redis_storage'
@@ -0,0 +1,40 @@
1
+ module Verdict
2
+ module Storage
3
+ class MemoryStorage
4
+ attr_reader :assignments, :start_timestamps
5
+
6
+ def initialize
7
+ @assignments = {}
8
+ @start_timestamps = {}
9
+ end
10
+
11
+ def store_assignment(assignment)
12
+ @assignments[assignment.experiment.handle] ||= {}
13
+ @assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
14
+ true
15
+ end
16
+
17
+ def retrieve_assignment(experiment, subject_identifier)
18
+ experiment_store = @assignments[experiment.handle] || {}
19
+ experiment_store[subject_identifier]
20
+ end
21
+
22
+ def remove_assignment(experiment, subject_identifier)
23
+ @assignments[assignment.experiment.handle] ||= {}
24
+ @assignments[assignment.experiment.handle].delete(subject_identifier)
25
+ end
26
+
27
+ def clear_experiment(experiment)
28
+ @assignments.delete(experiment.handle)
29
+ end
30
+
31
+ def retrieve_start_timestamp(experiment)
32
+ @start_timestamps[experiment.handle]
33
+ end
34
+
35
+ def store_start_timestamp(experiment, timestamp)
36
+ @start_timestamps[experiment.handle] = timestamp
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,38 @@
1
+ module Verdict
2
+ module Storage
3
+ class MockStorage
4
+ # Should store the assignments to allow quick lookups.
5
+ # - Assignments should be unique on the combination of
6
+ # `assignment.experiment.handle` and `assignment.subject_identifier`.
7
+ # - The main property to store is `group.handle`
8
+ # - Should return true if stored successfully.
9
+ def store_assignment(assignment)
10
+ false
11
+ end
12
+
13
+ # Should do a fast lookup of an assignment of the subject for the given experiment.
14
+ # - Should return nil if not found in store
15
+ # - Should return an Assignment instance otherwise.
16
+ def retrieve_assignment(experiment, subject_identifier)
17
+ nil
18
+ end
19
+
20
+ # Should remove the subject from storage, so it will be reassigned later.
21
+ def remove_assignment(experiment, subject_identifier)
22
+ end
23
+
24
+ # Should clear out the storage used for this experiment
25
+ def clear_experiment(experiment)
26
+ end
27
+
28
+ # Retrieves the start timestamp of the experiment
29
+ def retrieve_start_timestamp(experiment)
30
+ nil
31
+ end
32
+
33
+ # Stores the timestamp on which the experiment was started
34
+ def store_start_timestamp(experiment, timestamp)
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,62 @@
1
+ module Verdict
2
+ module Storage
3
+ class RedisStorage
4
+ attr_accessor :redis, :key_prefix
5
+
6
+ def initialize(redis = nil, options = {})
7
+ @redis = redis
8
+ @key_prefix = options[:key_prefix] || 'experiments/'
9
+ end
10
+
11
+ def retrieve_assignment(experiment, subject_identifier)
12
+ if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
13
+ hash = JSON.parse(value)
14
+ experiment.subject_assignment(
15
+ subject_identifier,
16
+ experiment.group(hash['group']),
17
+ DateTime.parse(hash['created_at']).to_time
18
+ )
19
+ end
20
+ rescue ::Redis::BaseError => e
21
+ raise Verdict::StorageError, "Redis error: #{e.message}"
22
+ end
23
+
24
+ def store_assignment(assignment)
25
+ hash = { group: assignment.handle, created_at: assignment.created_at }
26
+ redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
27
+ rescue ::Redis::BaseError => e
28
+ raise Verdict::StorageError, "Redis error: #{e.message}"
29
+ end
30
+
31
+ def remove_assignment(experiment, subject_identifier)
32
+ redis.hdel(generate_experiment_key(experiment), subject_identifier)
33
+ end
34
+
35
+ def clear_experiment(experiment)
36
+ redis.del(generate_experiment_key(experiment))
37
+ redis.del(generate_experiment_start_timestamp_key(experiment))
38
+ end
39
+
40
+ def retrieve_start_timestamp(experiment)
41
+ if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
42
+ DateTime.parse(started_at).to_time
43
+ end
44
+ end
45
+
46
+ def store_start_timestamp(experiment, timestamp)
47
+ redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
48
+ end
49
+
50
+
51
+ private
52
+
53
+ def generate_experiment_key(experiment)
54
+ "#{@key_prefix}#{experiment.handle}"
55
+ end
56
+
57
+ def generate_experiment_start_timestamp_key(experiment)
58
+ "#{@key_prefix}#{experiment.handle}/started_at"
59
+ end
60
+ end
61
+ end
62
+ end
@@ -5,11 +5,11 @@ def require_env(key)
5
5
  end
6
6
 
7
7
  namespace :experiments do
8
-
8
+
9
9
  desc "List all defined experiments"
10
10
  task :list => 'environment' do
11
- length = Experiments.repository.keys.map(&:length).max
12
- Experiments.repository.each do |_, experiment|
11
+ length = Verdict.repository.keys.map(&:length).max
12
+ Verdict.repository.each do |_, experiment|
13
13
  print "#{experiment.handle.ljust(length)} | "
14
14
  print "Groups: #{experiment.groups.values.map(&:to_s).join(', ')}"
15
15
  puts
@@ -18,7 +18,7 @@ namespace :experiments do
18
18
 
19
19
  desc "Looks up the assignment for a given experiment and subject"
20
20
  task :lookup_assignment => 'environment' do
21
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
21
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
22
22
  subject_identifier = require_env('subject')
23
23
  assignment = experiment.lookup_assignment_for_identifier(subject_identifier)
24
24
  if assignment.nil?
@@ -32,7 +32,7 @@ namespace :experiments do
32
32
 
33
33
  desc "Manually assign a subject to a given group in an experiment"
34
34
  task :assign_manually => 'environment' do
35
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
35
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
36
36
  group = experiment.group(require_env('group')) or raise "Group not found"
37
37
  assignment = experiment.subject_assignment(require_env('subject'), group, false)
38
38
  experiment.store_assignment(assignment)
@@ -40,20 +40,20 @@ namespace :experiments do
40
40
 
41
41
  desc "Disqualify a subject from an experiment"
42
42
  task :disqualify => 'environment' do
43
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
43
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
44
44
  assignment = experiment.subject_assignment(require_env('subject'), nil, false)
45
45
  experiment.store_assignment(assignment)
46
46
  end
47
47
 
48
48
  desc "Removes the assignment for a subject so it will be reassigned to the experiment."
49
49
  task :remove_assignment => 'environment' do
50
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
50
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
51
51
  experiment.remove_subject_identifier(require_env('subject'))
52
52
  end
53
53
 
54
54
  desc "Runs the cleanup tasks for an experiment"
55
55
  task :wrapup => 'environment' do
56
- experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
56
+ experiment = Verdict[require_env('experiment')] or raise "Experiment not found"
57
57
  experiment.wrapup
58
58
  end
59
59
  end
@@ -1,3 +1,3 @@
1
1
  module Verdict
2
- VERSION = "0.2.0"
2
+ VERSION = "0.2.1"
3
3
  end
@@ -1,7 +1,7 @@
1
1
  require 'test_helper'
2
2
  require 'json'
3
3
 
4
- class AssignmentTest < MiniTest::Unit::TestCase
4
+ class AssignmentTest < Minitest::Test
5
5
 
6
6
  def setup
7
7
  @experiment = Verdict::Experiment.new('assignment test')
@@ -11,8 +11,8 @@ class AssignmentTest < MiniTest::Unit::TestCase
11
11
  def test_basic_properties
12
12
  assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
13
13
  assert_equal 'test_subject_id', assignment.subject_identifier
14
- assert_equal @experiment, assignment.experiment
15
- assert_equal @group, assignment.group
14
+ assert_equal @experiment, assignment.experiment
15
+ assert_equal @group, assignment.group
16
16
  assert assignment.returning?
17
17
  assert assignment.qualified?
18
18
  assert_equal :control, assignment.to_sym
@@ -20,7 +20,7 @@ class AssignmentTest < MiniTest::Unit::TestCase
20
20
  assert_kind_of Time, assignment.created_at
21
21
 
22
22
  non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
23
- assert_equal nil, non_assignment.group
23
+ assert_equal nil, non_assignment.group
24
24
  assert !non_assignment.returning?
25
25
  assert !non_assignment.qualified?
26
26
  assert_equal nil, non_assignment.to_sym
@@ -1,7 +1,7 @@
1
1
  require 'test_helper'
2
2
  require 'json'
3
3
 
4
- class ConversionTest < MiniTest::Unit::TestCase
4
+ class ConversionTest < Minitest::Test
5
5
 
6
6
  def setup
7
7
  @experiment = Verdict::Experiment.new('conversion test') do
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class EventLoggerTest < MiniTest::Unit::TestCase
3
+ class EventLoggerTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @experiment = Verdict::Experiment.new(:logger) do
@@ -1,7 +1,7 @@
1
1
  require 'json'
2
2
  require 'test_helper'
3
3
 
4
- class ExperimentTest < MiniTest::Unit::TestCase
4
+ class ExperimentTest < Minitest::Test
5
5
 
6
6
  def test_no_qualifier
7
7
  e = Verdict::Experiment.new('test')
@@ -109,7 +109,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
109
109
  mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
110
110
  mock_store.expects(:store_assignment).never
111
111
  e.assign(mock('subject'))
112
- end
112
+ end
113
113
 
114
114
  def test_new_unqualified_assignment_with_store_unqualified
115
115
  mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
@@ -202,7 +202,7 @@ class ExperimentTest < MiniTest::Unit::TestCase
202
202
 
203
203
  conversion = e.convert(subject, :my_goal)
204
204
  assert_equal 'test_subject', conversion.subject_identifier
205
- assert_equal :my_goal, conversion.goal
205
+ assert_equal :my_goal, conversion.goal
206
206
  end
207
207
 
208
208
  def test_json
@@ -309,4 +309,13 @@ class ExperimentTest < MiniTest::Unit::TestCase
309
309
  e.assign(stub(id: '123'))
310
310
  assert e.started?, "The experiment should have started after the first assignment"
311
311
  end
312
+
313
+ def test_no_storage
314
+ e = Verdict::Experiment.new('starting_test') do
315
+ groups { group :all, 100 }
316
+ storage :none
317
+ end
318
+
319
+ assert_kind_of Verdict::Storage::MockStorage, e.storage
320
+ end
312
321
  end
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class ExperimentTest < MiniTest::Unit::TestCase
3
+ class ExperimentTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  Verdict.repository.clear
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class GroupTest < MiniTest::Unit::TestCase
3
+ class GroupTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @experiment = Verdict::Experiment.new('a')
@@ -38,5 +38,5 @@ class GroupTest < MiniTest::Unit::TestCase
38
38
  assert_equal 'control', json['handle']
39
39
  assert_equal 'testing', json['metadata']['name']
40
40
  assert_equal 'description', json['metadata']['description']
41
- end
41
+ end
42
42
  end