verdict 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,43 @@
1
+ require 'test_helper'
2
+
3
+ class EventLoggerTest < MiniTest::Unit::TestCase
4
+
5
+ def setup
6
+ @experiment = Verdict::Experiment.new(:logger) do
7
+ groups { group :all, 100}
8
+ end
9
+
10
+ @logger = mock('logger')
11
+ @event_logger = Verdict::EventLogger.new(@logger, :debug)
12
+ end
13
+
14
+ def test_log_unqualified_returning_assignment
15
+ @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=returning qualified=false')
16
+ assignment = Verdict::Assignment.new(@experiment, 'subject', nil, Time.now)
17
+ @event_logger.log_assignment(assignment)
18
+ end
19
+
20
+ def test_log_unqualified_new_assignment
21
+ @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=new qualified=false')
22
+ assignment = Verdict::Assignment.new(@experiment, 'subject', nil, nil)
23
+ @event_logger.log_assignment(assignment)
24
+ end
25
+
26
+ def test_log_qualified_returning_assignment
27
+ @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=returning qualified=true group=all')
28
+ assignment = Verdict::Assignment.new(@experiment, 'subject', @experiment.group(:all), Time.now)
29
+ @event_logger.log_assignment(assignment)
30
+ end
31
+
32
+ def test_log_qualified_new_assignment
33
+ @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=new qualified=true group=all')
34
+ assignment = Verdict::Assignment.new(@experiment, 'subject', @experiment.group(:all), nil)
35
+ @event_logger.log_assignment(assignment)
36
+ end
37
+
38
+ def test_log_conversion
39
+ @logger.expects(:debug).with('[Verdict::Conversion] experiment=logger subject=subject goal=my_goal')
40
+ conversion = Verdict::Conversion.new(@experiment, 'subject', :my_goal)
41
+ @event_logger.log_conversion(conversion)
42
+ end
43
+ end
@@ -0,0 +1,280 @@
1
+ require 'json'
2
+ require 'test_helper'
3
+
4
+ class ExperimentTest < MiniTest::Unit::TestCase
5
+
6
+ def test_no_qualifier
7
+ e = Verdict::Experiment.new('test')
8
+ assert !e.has_qualifier?
9
+ assert e.everybody_qualifies?
10
+ end
11
+
12
+ def test_qualifier
13
+ e = Verdict::Experiment.new('test') do |experiment|
14
+ qualify { |subject| subject.country == 'CA' }
15
+ groups do
16
+ group :all, 100
17
+ end
18
+ end
19
+
20
+ assert e.has_qualifier?
21
+ assert !e.everybody_qualifies?
22
+
23
+ subject_stub = Struct.new(:id, :country)
24
+ ca_subject = subject_stub.new(1, 'CA')
25
+ us_subject = subject_stub.new(1, 'US')
26
+
27
+ assert e.qualifier.call(ca_subject)
28
+ assert !e.qualifier.call(us_subject)
29
+
30
+ qualified = e.assign(ca_subject)
31
+ assert_kind_of Verdict::Assignment, qualified
32
+ assert_equal e.group(:all), qualified.group
33
+
34
+ non_qualified = e.assign(us_subject)
35
+ assert_kind_of Verdict::Assignment, non_qualified
36
+ assert !non_qualified.qualified?
37
+ assert_equal nil, non_qualified.group
38
+ end
39
+
40
+ def test_assignment
41
+ e = Verdict::Experiment.new('test') do
42
+ qualify { |subject| subject <= 2 }
43
+ groups do
44
+ group :a, :half
45
+ group :b, :rest
46
+ end
47
+ end
48
+
49
+ assignment = e.assign(1)
50
+ assert_kind_of Verdict::Assignment, assignment
51
+ assert assignment.qualified?
52
+ assert !assignment.returning?
53
+ assert_equal assignment.group, e.group(:a)
54
+
55
+ assignment = e.assign(3)
56
+ assert_kind_of Verdict::Assignment, assignment
57
+ assert !assignment.qualified?
58
+
59
+ assert_equal :a, e.switch(1)
60
+ assert_equal :b, e.switch(2)
61
+ assert_equal nil, e.switch(3)
62
+ end
63
+
64
+ def test_subject_identifier
65
+ e = Verdict::Experiment.new('test')
66
+ assert_equal '123', e.retrieve_subject_identifier(stub(id: 123, to_s: '456'))
67
+ assert_equal '456', e.retrieve_subject_identifier(stub(to_s: '456'))
68
+ assert_raises(Verdict::EmptySubjectIdentifier) { e.retrieve_subject_identifier(stub(id: nil)) }
69
+ assert_raises(Verdict::EmptySubjectIdentifier) { e.retrieve_subject_identifier(stub(to_s: '')) }
70
+ end
71
+
72
+ def test_new_unqualified_assignment_without_store_unqualified
73
+ mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
74
+ e = Verdict::Experiment.new('test') do
75
+ qualify { mock_qualifier.qualifies? }
76
+ storage mock_store, store_unqualified: false
77
+ end
78
+
79
+ mock_qualifier.expects(:qualifies?).returns(false)
80
+ mock_store.expects(:retrieve_assignment).never
81
+ mock_store.expects(:store_assignment).never
82
+ e.assign(mock('subject'))
83
+ end
84
+
85
+ def test_returning_qualified_assignment_without_store_unqualified
86
+ mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
87
+ e = Verdict::Experiment.new('test') do
88
+ qualify { mock_qualifier.qualifies? }
89
+ storage mock_store, store_unqualified: false
90
+ groups { group :all, 100 }
91
+ end
92
+
93
+ qualified_assignment = e.subject_assignment(mock('identifier'), e.group(:all), Time.now)
94
+ mock_qualifier.expects(:qualifies?).returns(true)
95
+ mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
96
+ mock_store.expects(:store_assignment).never
97
+ e.assign(mock('subject'))
98
+ end
99
+
100
+ def test_new_unqualified_assignment_with_store_unqualified
101
+ mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
102
+ e = Verdict::Experiment.new('test') do
103
+ qualify { mock_qualifier.qualifies? }
104
+ storage mock_store, store_unqualified: true
105
+ end
106
+
107
+ mock_qualifier.expects(:qualifies?).returns(false)
108
+ mock_store.expects(:retrieve_assignment).returns(nil).once
109
+ mock_store.expects(:store_assignment).once
110
+ e.assign(mock('subject'))
111
+ end
112
+
113
+ def test_returning_unqualified_assignment_with_store_unqualified
114
+ mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
115
+ e = Verdict::Experiment.new('test') do
116
+ qualify { mock_qualifier.qualifies? }
117
+ storage mock_store, store_unqualified: true
118
+ end
119
+
120
+ unqualified_assignment = e.subject_assignment(mock('subject_identifier'), nil, Time.now)
121
+ mock_qualifier.expects(:qualifies?).never
122
+ mock_store.expects(:retrieve_assignment).returns(unqualified_assignment).once
123
+ mock_store.expects(:store_assignment).never
124
+ e.assign(mock('subject'))
125
+ end
126
+
127
+ def test_returning_qualified_assignment_with_store_unqualified
128
+ mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
129
+ e = Verdict::Experiment.new('test') do
130
+ qualify { mock_qualifier.qualifies? }
131
+ storage mock_store, store_unqualified: true
132
+ groups { group :all, 100 }
133
+ end
134
+
135
+ qualified_assignment = e.subject_assignment(mock('subject_identifier'), e.group(:all), Time.now)
136
+ mock_qualifier.expects(:qualifies?).never
137
+ mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
138
+ mock_store.expects(:store_assignment).never
139
+ e.assign(mock('subject'))
140
+ end
141
+
142
+ def test_disqualify
143
+ e = Verdict::Experiment.new('test') do
144
+ groups { group :all, 100 }
145
+ end
146
+
147
+ subject = stub(id: 'walrus')
148
+ original_assignment = e.assign(subject)
149
+ assert original_assignment.qualified?
150
+ new_assignment = e.disqualify(subject)
151
+ assert !new_assignment.qualified?
152
+ end
153
+
154
+ def test_assignment_event_logging
155
+ e = Verdict::Experiment.new('test') do
156
+ groups { group :all, 100 }
157
+ end
158
+
159
+ e.stubs(:event_logger).returns(logger = mock('event_logger'))
160
+ logger.expects(:log_assignment).with(kind_of(Verdict::Assignment))
161
+
162
+ e.assign(stub(id: 'subject_identifier'))
163
+ end
164
+
165
+ def test_conversion_event_logging
166
+ e = Verdict::Experiment.new('test')
167
+
168
+ e.stubs(:event_logger).returns(logger = mock('logger'))
169
+ logger.expects(:log_conversion).with(kind_of(Verdict::Conversion))
170
+
171
+ conversion = e.convert(subject = stub(id: 'test_subject'), :my_goal)
172
+ assert_equal 'test_subject', conversion.subject_identifier
173
+ assert_equal :my_goal, conversion.goal
174
+ end
175
+
176
+ def test_json
177
+ e = Verdict::Experiment.new(:json) do
178
+ name 'testing'
179
+ subject_type 'visitor'
180
+ groups do
181
+ group :a, :half
182
+ group :b, :rest
183
+ end
184
+ end
185
+
186
+ Timecop.freeze(Time.new(2013, 2, 3, 4, 5, 6, '+00:00')) do
187
+ e.send(:ensure_experiment_has_started)
188
+ end
189
+
190
+ json = JSON.parse(e.to_json)
191
+ assert_equal 'json', json['handle']
192
+ assert_equal false, json['has_qualifier']
193
+ assert_kind_of Enumerable, json['groups']
194
+ assert_equal 'testing', json['metadata']['name']
195
+ assert_equal 'visitor', json['subject_type']
196
+ assert_equal '2013-02-03T04:05:06Z', json['started_at']
197
+ end
198
+
199
+ def test_storage_read_failure
200
+ storage_mock = Verdict::Storage::MockStorage.new
201
+ e = Verdict::Experiment.new(:json) do
202
+ groups { group :all, 100 }
203
+ storage storage_mock
204
+ end
205
+
206
+ storage_mock.stubs(:retrieve_assignment).raises(Verdict::StorageError, 'storage read issues')
207
+ rescued_assignment = e.assign(stub(id: 123))
208
+ assert !rescued_assignment.qualified?
209
+ end
210
+
211
+ def test_storage_write_failure
212
+ storage_mock = Verdict::Storage::MockStorage.new
213
+ e = Verdict::Experiment.new(:json) do
214
+ groups { group :all, 100 }
215
+ storage storage_mock
216
+ end
217
+
218
+ storage_mock.expects(:retrieve_assignment).returns(e.subject_assignment(mock('subject_identifier'), e.group(:all)))
219
+ storage_mock.expects(:store_assignment).raises(Verdict::StorageError, 'storage write issues')
220
+ rescued_assignment = e.assign(stub(id: 456))
221
+ assert !rescued_assignment.qualified?
222
+ end
223
+
224
+ def test_initial_started_at
225
+ e = Verdict::Experiment.new('test') do
226
+ groups { group :all, 100 }
227
+ end
228
+
229
+ e.subject_storage.expects(:retrieve_start_timestamp).returns(nil)
230
+ e.subject_storage.expects(:store_start_timestamp).once
231
+ e.send(:ensure_experiment_has_started)
232
+ end
233
+
234
+ def test_subsequent_started_at_when_start_time_is_memoized
235
+ e = Verdict::Experiment.new('test') do
236
+ groups { group :all, 100 }
237
+ end
238
+
239
+ e.send(:ensure_experiment_has_started)
240
+ e.subject_storage.expects(:retrieve_start_timestamp).never
241
+ e.subject_storage.expects(:store_start_timestamp).never
242
+ e.send(:ensure_experiment_has_started)
243
+ end
244
+
245
+ def test_subsequent_started_at_when_start_time_is_not_memoized
246
+ e = Verdict::Experiment.new('test') do
247
+ groups { group :all, 100 }
248
+ end
249
+
250
+ e.subject_storage.expects(:retrieve_start_timestamp).returns(Time.now.utc)
251
+ e.subject_storage.expects(:store_start_timestamp).never
252
+ e.send(:ensure_experiment_has_started)
253
+ end
254
+
255
+ def test_qualify_based_on_experiment_start_timestamp
256
+ Timecop.freeze(Time.new(2012)) do
257
+ e = Verdict::Experiment.new('test') do
258
+ qualify { |subject| subject.created_at >= self.started_at }
259
+ groups { group :all, 100 }
260
+ end
261
+
262
+ subject = stub(id: 'old', created_at: Time.new(2011))
263
+ assert !e.assign(subject).qualified?
264
+
265
+ subject = stub(id: 'new', created_at: Time.new(2013))
266
+ assert e.assign(subject).qualified?
267
+ end
268
+ end
269
+
270
+ def test_experiment_starting_behavior
271
+ e = Verdict::Experiment.new('starting_test') do
272
+ groups { group :all, 100 }
273
+ end
274
+
275
+ assert !e.started?, "The experiment should not have started yet"
276
+
277
+ e.assign(stub(id: '123'))
278
+ assert e.started?, "The experiment should have started after the first assignment"
279
+ end
280
+ end
@@ -0,0 +1,32 @@
1
+ require 'test_helper'
2
+
3
+ class ExperimentTest < MiniTest::Unit::TestCase
4
+
5
+ def setup
6
+ Verdict.repository.clear
7
+ end
8
+
9
+ def test_should_keep_list_of_all_experiments
10
+ size_at_start = Verdict.repository.size
11
+ e = Verdict::Experiment.define('test')
12
+
13
+ assert_equal size_at_start + 1, Verdict.repository.size
14
+ assert_equal e, Verdict['test']
15
+ end
16
+
17
+ def test_should_not_allow_experiments_with_the_same_name
18
+ Verdict::Experiment.define('test_duplicate')
19
+ assert_raises(Verdict::ExperimentHandleNotUnique) do
20
+ Verdict::Experiment.define('test_duplicate')
21
+ end
22
+ end
23
+
24
+ def test_json_export
25
+ e1 = Verdict::Experiment.define('test_1') { groups { group :all, 100 } }
26
+ e2 = Verdict::Experiment.define('test_2') { groups { group :all, 100 } }
27
+
28
+ json = JSON.parse(Verdict.repository.to_json)
29
+ assert_equal ['test_1', 'test_2'], json.keys
30
+ assert_equal json['test_1'], JSON.parse(e1.to_json)
31
+ end
32
+ end
@@ -0,0 +1,42 @@
1
+ require 'test_helper'
2
+
3
+ class GroupTest < MiniTest::Unit::TestCase
4
+
5
+ def setup
6
+ @experiment = Verdict::Experiment.new('a')
7
+ end
8
+
9
+ def test_basic_properties
10
+ group = Verdict::Group.new(@experiment, :test)
11
+
12
+ assert_equal @experiment, group.experiment
13
+ assert_kind_of Verdict::Group, group
14
+ assert_equal 'test', group.handle
15
+ assert_equal 'test', group.to_s
16
+ assert_equal :test, group.to_sym
17
+ end
18
+
19
+ def test_triple_equals
20
+ group = Verdict::Group.new(@experiment, 'control')
21
+ assert group === Verdict::Group.new(@experiment, :control)
22
+ assert group === 'control'
23
+ assert group === :control
24
+ assert !(group === nil)
25
+
26
+ assert !(group === Verdict::Group.new(@experiment, :test))
27
+ assert !(group === Verdict::Group.new(Verdict::Experiment.new('b'), :test))
28
+ assert !(group === 'test')
29
+ assert !(group === nil)
30
+ end
31
+
32
+ def test_json
33
+ group = Verdict::Group.new(@experiment, 'control')
34
+ group.name 'testing'
35
+ group.description 'description'
36
+
37
+ json = JSON.parse(group.to_json)
38
+ assert_equal 'control', json['handle']
39
+ assert_equal 'testing', json['metadata']['name']
40
+ assert_equal 'description', json['metadata']['description']
41
+ end
42
+ end
@@ -0,0 +1,45 @@
1
+ require 'test_helper'
2
+
3
+ class MemorySubjectStorageTest < MiniTest::Unit::TestCase
4
+
5
+ def setup
6
+ @storage = storage = Verdict::Storage::MemoryStorage.new
7
+ @experiment = Verdict::Experiment.new(:memory_storage) do
8
+ groups { group :all, 100 }
9
+ storage storage, store_unqualified: true
10
+ end
11
+
12
+ @subject = stub(id: 'bootscale')
13
+ end
14
+
15
+ def test_wrapup
16
+ @experiment.assign(@subject)
17
+ @experiment.wrapup
18
+ assert @experiment.lookup(@subject).nil?
19
+ end
20
+
21
+ def test_with_memory_store
22
+ assignment_1 = @experiment.assign(@subject)
23
+ assignment_2 = @experiment.assign(@subject)
24
+ assert !assignment_1.returning?
25
+ assert assignment_2.returning?
26
+ end
27
+
28
+ def test_assignment_lookup
29
+ assert @experiment.lookup(@subject).nil?
30
+ @experiment.assign(@subject)
31
+ assert !@experiment.lookup(@subject).nil?
32
+ end
33
+
34
+ def test_remove_assignment
35
+ assert !@experiment.assign(@subject).returning?
36
+ @experiment.wrapup
37
+ assert !@experiment.assign(@subject).returning?
38
+ end
39
+
40
+ def test_started_at
41
+ assert @storage.start_timestamps[@experiment.handle].nil?
42
+ @experiment.send(:ensure_experiment_has_started)
43
+ assert @storage.start_timestamps[@experiment.handle].instance_of?(Time)
44
+ end
45
+ end
@@ -0,0 +1,44 @@
1
+ require 'test_helper'
2
+
3
+ class MetadataTest < MiniTest::Unit::TestCase
4
+
5
+ def test_experiment_metadata
6
+ experiment = Verdict::Experiment.new('experiment metadata') do
7
+ name "Metadata test"
8
+ description "For testing metadata functionality"
9
+ owner "Willem van Bergen"
10
+ end
11
+
12
+ assert_equal "Metadata test", experiment.name
13
+ assert_equal "For testing metadata functionality", experiment.description
14
+ assert_equal "Willem van Bergen", experiment.owner
15
+
16
+ assert_equal experiment.metadata, {
17
+ :name => 'Metadata test',
18
+ :description => 'For testing metadata functionality',
19
+ :owner => 'Willem van Bergen'
20
+ }
21
+ end
22
+
23
+ def test_group_metadata
24
+ experiment = Verdict::Experiment.new('group metadata') do
25
+ groups do
26
+ group :all, 100 do
27
+ name "Group metadata test"
28
+ description "For testing metadata functionality"
29
+ screenshot "http://example.com/image.png"
30
+ end
31
+ end
32
+ end
33
+
34
+ assert_equal "Group metadata test", experiment.group(:all).name
35
+ assert_equal "For testing metadata functionality", experiment.group(:all).description
36
+ assert_equal "http://example.com/image.png", experiment.group(:all).screenshot
37
+
38
+ assert_equal experiment.group(:all).metadata, {
39
+ :name => 'Group metadata test',
40
+ :description => 'For testing metadata functionality',
41
+ :screenshot => 'http://example.com/image.png'
42
+ }
43
+ end
44
+ end