verdict 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -0
  3. data/Gemfile +4 -0
  4. data/README.md +56 -33
  5. data/lib/verdict.rb +2 -1
  6. data/lib/verdict/experiment.rb +15 -7
  7. data/lib/verdict/segmenters.rb +4 -0
  8. data/lib/verdict/segmenters/base_segmenter.rb +84 -0
  9. data/lib/verdict/segmenters/fixed_percentage_segmenter.rb +56 -0
  10. data/lib/verdict/segmenters/rollout_segmenter.rb +55 -0
  11. data/lib/verdict/segmenters/static_segmenter.rb +27 -0
  12. data/lib/verdict/storage.rb +3 -138
  13. data/lib/verdict/storage/memory_storage.rb +40 -0
  14. data/lib/verdict/storage/mock_storage.rb +38 -0
  15. data/lib/verdict/storage/redis_storage.rb +62 -0
  16. data/lib/verdict/tasks.rake +8 -8
  17. data/lib/verdict/version.rb +1 -1
  18. data/test/assignment_test.rb +4 -4
  19. data/test/conversion_test.rb +1 -1
  20. data/test/event_logger_test.rb +1 -1
  21. data/test/experiment_test.rb +12 -3
  22. data/test/experiments_repository_test.rb +1 -1
  23. data/test/group_test.rb +2 -2
  24. data/test/metadata_test.rb +1 -1
  25. data/test/{fixed_percentage_segmenter_test.rb → segmenters/fixed_percentage_segmenter_test.rb} +11 -11
  26. data/test/{rollout_segmenter_test.rb → segmenters/rollout_segmenter_test.rb} +2 -2
  27. data/test/{static_segmenter_test.rb → segmenters/static_segmenter_test.rb} +2 -2
  28. data/test/{memory_subject_storage_test.rb → storage/memory_subject_storage_test.rb} +1 -1
  29. data/test/{redis_subject_storage_test.rb → storage/redis_subject_storage_test.rb} +2 -2
  30. data/test/test_helper.rb +8 -0
  31. data/verdict.gemspec +6 -6
  32. metadata +24 -19
  33. data/lib/verdict/fixed_percentage_segmenter.rb +0 -52
  34. data/lib/verdict/rollout_segmenter.rb +0 -53
  35. data/lib/verdict/segmenter.rb +0 -87
  36. data/lib/verdict/static_segmenter.rb +0 -24
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class MetadataTest < MiniTest::Unit::TestCase
3
+ class MetadataTest < Minitest::Test
4
4
 
5
5
  def test_experiment_metadata
6
6
  experiment = Verdict::Experiment.new('experiment metadata') do
@@ -1,9 +1,9 @@
1
1
  require 'test_helper'
2
2
 
3
- class FixedPercentageSegmenterTest < MiniTest::Unit::TestCase
3
+ class FixedPercentageSegmenterTest < Minitest::Test
4
4
 
5
5
  def test_add_up_to_100_percent
6
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
6
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
7
7
  s.group :segment1, 1
8
8
  s.group :segment2, 54
9
9
  s.group :segment3, 27
@@ -18,7 +18,7 @@ class FixedPercentageSegmenterTest < MiniTest::Unit::TestCase
18
18
  end
19
19
 
20
20
  def test_definition_ofhalf_and_rest
21
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
21
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
22
22
  s.group :first_half, :half
23
23
  s.group :second_half, :rest
24
24
  s.verify!
@@ -30,7 +30,7 @@ class FixedPercentageSegmenterTest < MiniTest::Unit::TestCase
30
30
 
31
31
  def test_raises_if_less_than_100_percent
32
32
  assert_raises(Verdict::SegmentationError) do
33
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
33
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
34
34
  s.group :too_little, 99
35
35
  s.verify!
36
36
  end
@@ -38,34 +38,34 @@ class FixedPercentageSegmenterTest < MiniTest::Unit::TestCase
38
38
 
39
39
  def test_raises_if_greather_than_100_percent
40
40
  assert_raises(Verdict::SegmentationError) do
41
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
41
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
42
42
  s.group :too_much, 101
43
43
  s.verify!
44
44
  end
45
45
  end
46
46
 
47
47
  def test_consistent_assignment_for_subjects
48
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
48
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
49
49
  s.group :first_half, :half
50
50
  s.group :second_half, :rest
51
51
  s.verify!
52
52
 
53
- 3.times do
53
+ 3.times do
54
54
  assert s.groups['first_half'] === s.assign(1, nil, nil)
55
55
  assert s.groups['second_half'] === s.assign(2, nil, nil)
56
56
  end
57
57
  end
58
58
 
59
59
  def test_fair_segmenting
60
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
60
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
61
61
  s.group :first_third, 33
62
62
  s.group :second_third, 33
63
63
  s.group :final_third, :rest
64
64
  s.verify!
65
65
 
66
66
  assignments = { :first_third => 0, :second_third => 0, :final_third => 0 }
67
- 200.times do |n|
68
- assignment = s.assign(n, nil, nil)
67
+ 200.times do |n|
68
+ assignment = s.assign(n, nil, nil)
69
69
  assignments[assignment.to_sym] += 1
70
70
  end
71
71
 
@@ -76,7 +76,7 @@ class FixedPercentageSegmenterTest < MiniTest::Unit::TestCase
76
76
  end
77
77
 
78
78
  def test_group_json_export
79
- s = Verdict::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
79
+ s = Verdict::Segmenters::FixedPercentageSegmenter.new(Verdict::Experiment.new('test'))
80
80
  s.group :first_third, 33
81
81
  s.group :rest, :rest
82
82
  s.verify!
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class RolloutPercentageSegmenterTest < MiniTest::Unit::TestCase
3
+ class RolloutPercentageSegmenterTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @experiment = Verdict::Experiment.new('test') do
@@ -11,7 +11,7 @@ class RolloutPercentageSegmenterTest < MiniTest::Unit::TestCase
11
11
  def test_assignment
12
12
  included_subject = stub(id: 1)
13
13
  excluded_subject = stub(id: 2)
14
-
14
+
15
15
  included_assignment = @experiment.assign(included_subject)
16
16
  assert included_assignment.qualified?
17
17
  assert included_assignment.permanent?
@@ -1,9 +1,9 @@
1
1
  require 'test_helper'
2
2
 
3
- class StaticSegmenterTest < MiniTest::Unit::TestCase
3
+ class StaticSegmenterTest < Minitest::Test
4
4
 
5
5
  def setup
6
- @segmenter = Verdict::StaticSegmenter.new(Verdict::Experiment.new('test'))
6
+ @segmenter = Verdict::Segmenters::StaticSegmenter.new(Verdict::Experiment.new('test'))
7
7
  @segmenter.group :beta, ['id1', 'id2']
8
8
  end
9
9
 
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class MemorySubjectStorageTest < MiniTest::Unit::TestCase
3
+ class MemorySubjectStorageTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @storage = storage = Verdict::Storage::MemoryStorage.new
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class RedisSubjectStorageTest < MiniTest::Unit::TestCase
3
+ class RedisSubjectStorageTest < Minitest::Test
4
4
 
5
5
  def setup
6
6
  @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
@@ -80,7 +80,7 @@ class RedisSubjectStorageTest < MiniTest::Unit::TestCase
80
80
 
81
81
  def test_started_at
82
82
  key = @storage.send(:generate_experiment_start_timestamp_key, @experiment)
83
-
83
+
84
84
  assert !@redis.exists(key)
85
85
  a = @experiment.send(:ensure_experiment_has_started)
86
86
  assert @redis.exists(key)
@@ -1,3 +1,11 @@
1
+ unless RUBY_ENGINE == 'rbx'
2
+ require 'simplecov'
3
+ SimpleCov.start do
4
+ add_filter "/vendor/"
5
+ add_filter "/test/"
6
+ end
7
+ end
8
+
1
9
  require "bundler/setup"
2
10
  require "minitest/autorun"
3
11
  require "minitest/pride"
@@ -16,10 +16,10 @@ Gem::Specification.new do |gem|
16
16
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
17
17
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
18
  gem.require_paths = ["lib"]
19
-
20
- gem.add_development_dependency "minitest", '~> 4.2'
21
- gem.add_development_dependency "rake"
22
- gem.add_development_dependency "mocha"
23
- gem.add_development_dependency "timecop"
24
- gem.add_development_dependency "redis"
19
+
20
+ gem.add_development_dependency("minitest", '~> 5.2')
21
+ gem.add_development_dependency("rake")
22
+ gem.add_development_dependency("mocha")
23
+ gem.add_development_dependency("timecop")
24
+ gem.add_development_dependency("redis")
25
25
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: verdict
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shopify
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-21 00:00:00.000000000 Z
11
+ date: 2014-02-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: minitest
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ~>
18
18
  - !ruby/object:Gem::Version
19
- version: '4.2'
19
+ version: '5.2'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ~>
25
25
  - !ruby/object:Gem::Version
26
- version: '4.2'
26
+ version: '5.2'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -99,14 +99,18 @@ files:
99
99
  - lib/verdict/conversion.rb
100
100
  - lib/verdict/event_logger.rb
101
101
  - lib/verdict/experiment.rb
102
- - lib/verdict/fixed_percentage_segmenter.rb
103
102
  - lib/verdict/group.rb
104
103
  - lib/verdict/metadata.rb
105
104
  - lib/verdict/railtie.rb
106
- - lib/verdict/rollout_segmenter.rb
107
- - lib/verdict/segmenter.rb
108
- - lib/verdict/static_segmenter.rb
105
+ - lib/verdict/segmenters.rb
106
+ - lib/verdict/segmenters/base_segmenter.rb
107
+ - lib/verdict/segmenters/fixed_percentage_segmenter.rb
108
+ - lib/verdict/segmenters/rollout_segmenter.rb
109
+ - lib/verdict/segmenters/static_segmenter.rb
109
110
  - lib/verdict/storage.rb
111
+ - lib/verdict/storage/memory_storage.rb
112
+ - lib/verdict/storage/mock_storage.rb
113
+ - lib/verdict/storage/redis_storage.rb
110
114
  - lib/verdict/tasks.rake
111
115
  - lib/verdict/version.rb
112
116
  - test/assignment_test.rb
@@ -114,13 +118,13 @@ files:
114
118
  - test/event_logger_test.rb
115
119
  - test/experiment_test.rb
116
120
  - test/experiments_repository_test.rb
117
- - test/fixed_percentage_segmenter_test.rb
118
121
  - test/group_test.rb
119
- - test/memory_subject_storage_test.rb
120
122
  - test/metadata_test.rb
121
- - test/redis_subject_storage_test.rb
122
- - test/rollout_segmenter_test.rb
123
- - test/static_segmenter_test.rb
123
+ - test/segmenters/fixed_percentage_segmenter_test.rb
124
+ - test/segmenters/rollout_segmenter_test.rb
125
+ - test/segmenters/static_segmenter_test.rb
126
+ - test/storage/memory_subject_storage_test.rb
127
+ - test/storage/redis_subject_storage_test.rb
124
128
  - test/test_helper.rb
125
129
  - verdict.gemspec
126
130
  homepage: http://github.com/Shopify/verdict
@@ -142,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
142
146
  version: '0'
143
147
  requirements: []
144
148
  rubyforge_project:
145
- rubygems_version: 2.0.14
149
+ rubygems_version: 2.0.3
146
150
  signing_key:
147
151
  specification_version: 4
148
152
  summary: A library to centrally define experiments for your application, and collect
@@ -153,11 +157,12 @@ test_files:
153
157
  - test/event_logger_test.rb
154
158
  - test/experiment_test.rb
155
159
  - test/experiments_repository_test.rb
156
- - test/fixed_percentage_segmenter_test.rb
157
160
  - test/group_test.rb
158
- - test/memory_subject_storage_test.rb
159
161
  - test/metadata_test.rb
160
- - test/redis_subject_storage_test.rb
161
- - test/rollout_segmenter_test.rb
162
- - test/static_segmenter_test.rb
162
+ - test/segmenters/fixed_percentage_segmenter_test.rb
163
+ - test/segmenters/rollout_segmenter_test.rb
164
+ - test/segmenters/static_segmenter_test.rb
165
+ - test/storage/memory_subject_storage_test.rb
166
+ - test/storage/redis_subject_storage_test.rb
163
167
  - test/test_helper.rb
168
+ has_rdoc:
@@ -1,52 +0,0 @@
1
- class Verdict::FixedPercentageSegmenter < Verdict::Segmenter
2
-
3
- class Group < Verdict::Group
4
-
5
- attr_reader :percentile_range
6
-
7
- def initialize(experiment, handle, percentile_range)
8
- super(experiment, handle)
9
- @percentile_range = percentile_range
10
- end
11
-
12
- def percentage_size
13
- percentile_range.end - percentile_range.begin
14
- end
15
-
16
- def to_s
17
- "#{handle} (#{percentage_size}%)"
18
- end
19
-
20
- def as_json(options = {})
21
- super(options).merge(percentage: percentage_size)
22
- end
23
- end
24
-
25
- def initialize(experiment)
26
- super
27
- @total_percentage_segmented = 0
28
- end
29
-
30
- def verify!
31
- raise Verdict::SegmentationError, "Should segment exactly 100% of the cases, but segments add up to #{@total_percentage_segmented}%." if @total_percentage_segmented != 100
32
- end
33
-
34
- def register_group(handle, size)
35
- percentage = size.kind_of?(Hash) && size[:percentage] ? size[:percentage] : size
36
- n = case percentage
37
- when :rest; 100 - @total_percentage_segmented
38
- when :half; 50
39
- when Integer; percentage
40
- else Integer(percentage)
41
- end
42
-
43
- group = Group.new(experiment, handle, @total_percentage_segmented ... (@total_percentage_segmented + n))
44
- @total_percentage_segmented += n
45
- return group
46
- end
47
-
48
- def assign(identifier, subject, context)
49
- percentile = Digest::MD5.hexdigest("#{@experiment.handle}#{identifier}").to_i(16) % 100
50
- groups.values.find { |group| group.percentile_range.include?(percentile) }
51
- end
52
- end
@@ -1,53 +0,0 @@
1
- class Verdict::RolloutSegmenter < Verdict::Segmenter
2
-
3
- class Group < Verdict::Group
4
-
5
- attr_reader :percentile_range
6
-
7
- def initialize(experiment, handle, percentile_range)
8
- super(experiment, handle)
9
- @percentile_range = percentile_range
10
- end
11
-
12
- def percentage_size
13
- percentile_range.end - percentile_range.begin
14
- end
15
-
16
- def to_s
17
- "#{handle} (#{percentage_size}%)"
18
- end
19
-
20
- def as_json(options = {})
21
- super(options).merge(percentage: percentage_size)
22
- end
23
- end
24
-
25
- def initialize(experiment)
26
- super
27
- @total_percentage_segmented = 0
28
- end
29
-
30
- def verify!
31
- raise Verdict::SegmentationError, "Should segment less than 100% of the cases, but segments add up to #{@total_percentage_segmented}%." if @total_percentage_segmented >= 100
32
- end
33
-
34
- def register_group(handle, size)
35
- percentage = size.kind_of?(Hash) && size[:percentage] ? size[:percentage] : size
36
- n = case percentage
37
- when :rest; 100 - @total_percentage_segmented
38
- when :half; 50
39
- when Integer; percentage
40
- else Integer(percentage)
41
- end
42
-
43
- group = Group.new(experiment, handle, @total_percentage_segmented ... (@total_percentage_segmented + n))
44
- @total_percentage_segmented += n
45
- return group
46
- end
47
-
48
- def assign(identifier, subject, context)
49
- percentile = Digest::MD5.hexdigest("#{@experiment.handle}#{identifier}").to_i(16) % 100
50
- groups.values.find { |group| group.percentile_range.include?(percentile) }
51
- end
52
-
53
- end
@@ -1,87 +0,0 @@
1
- require 'digest/md5'
2
-
3
- # Base class of all segmenters.
4
- #
5
- # The segmenter is responsible for assigning subjects to groups. You can
6
- # implement any assignment strategy you like by subclassing this class and
7
- # using it in your experiment.
8
- #
9
- # - You should implement the register_group method for the experiment definition DSL
10
- # to make the system aware of the groups that the segmenter could return.
11
- # - The verify! method is called after all the groups have been defined, so it can
12
- # detect internal inconsistencies in the group definitions.
13
- # - The assign method is where your assignment magic lives.
14
- class Verdict::Segmenter
15
-
16
- # The experiment to which this segmenter is associated
17
- attr_reader :experiment
18
-
19
- # A hash of the groups that are defined in this experiment, indexed by their
20
- # handle. The assign method should return one of the groups in this hash
21
- attr_reader :groups
22
-
23
- def initialize(experiment)
24
- @experiment = experiment
25
- @groups = {}
26
- end
27
-
28
- # DSL method to register a group. It calls the register_group method of the
29
- # segmenter implementation
30
- def group(handle, *args, &block)
31
- group = register_group(handle, *args)
32
- @groups[group.handle] = group
33
- group.instance_eval(&block) if block_given?
34
- end
35
-
36
- # The group method is called from the experiment definition DSL.
37
- # It should register a new group to the segmenter, with the given handle.
38
- #
39
- # - The handle parameter is a symbol that uniquely identifies the group within
40
- # this experiment.
41
- # - The return value of this method should be a Verdict::Group instance.
42
- def register_group(handle, *args)
43
- raise NotImplementedError
44
- end
45
-
46
- # The verify! method is called after all the groups have been defined in the
47
- # experiment definition DSL. You can run any consistency checks in this method,
48
- # and if anything is off, you can raise a Verdict::SegmentationError to
49
- # signify the problem.
50
- def verify!
51
- # noop by default
52
- end
53
-
54
- # The assign method is called to assign a subject to one of the groups that have been defined
55
- # in the segmenter implementation.
56
- #
57
- # - The identifier parameter is a string that uniquely identifies the subject.
58
- # - The subject paramater is the subject instance that was passed to the framework,
59
- # when the application code calls Experiment#assign or Experiment#switch.
60
- # - The context parameter is an object that was passed to the framework, you can use this
61
- # object any way you like in your segmenting logic.
62
- #
63
- # This method should return the Verdict::Group instance to which the subject should be assigned.
64
- # This instance should be one of the group instance that was registered in the definition DSL.
65
- def assign(identifier, subject, context)
66
- raise NotImplementedError
67
- end
68
-
69
-
70
- # This method is called whenever a subjects converts to a goal, i.e., when Experiment#convert
71
- # is called. You can use this to implement a feedback loop in your segmenter.
72
- #
73
- # - The identifier parameter is a string that uniquely identifies the subject.
74
- # - The subject paramater is the subject instance that was passed to the framework,
75
- # when the application code calls Experiment#assign or Experiment#switch.
76
- # - The conversion parameter is a Verdict::Conversion instance that describes what
77
- # goal the subject converted to.
78
- #
79
- # The return value of this method is not used.
80
- def conversion_feedback(identifier, subject, conversion)
81
- # noop by default
82
- end
83
- end
84
-
85
- require 'verdict/static_segmenter'
86
- require 'verdict/fixed_percentage_segmenter'
87
- require 'verdict/rollout_segmenter'