verdict 0.1.0

data/test/event_logger_test.rb

@@ -0,0 +1,43 @@
+require 'test_helper'
+
+class EventLoggerTest < MiniTest::Unit::TestCase
+
+  def setup
+    @experiment = Verdict::Experiment.new(:logger) do
+      groups { group :all, 100}
+    end
+
+    @logger = mock('logger')
+    @event_logger = Verdict::EventLogger.new(@logger, :debug)
+  end
+
+  def test_log_unqualified_returning_assignment
+    @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=returning qualified=false')
+    assignment = Verdict::Assignment.new(@experiment, 'subject', nil, Time.now)
+    @event_logger.log_assignment(assignment)
+  end
+
+  def test_log_unqualified_new_assignment
+    @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=new qualified=false')
+    assignment = Verdict::Assignment.new(@experiment, 'subject', nil, nil)
+    @event_logger.log_assignment(assignment)
+  end
+
+  def test_log_qualified_returning_assignment
+    @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=returning qualified=true group=all')
+    assignment = Verdict::Assignment.new(@experiment, 'subject', @experiment.group(:all), Time.now)
+    @event_logger.log_assignment(assignment)
+  end
+
+  def test_log_qualified_new_assignment
+    @logger.expects(:debug).with('[Verdict::Assignment] experiment=logger subject=subject status=new qualified=true group=all')
+    assignment = Verdict::Assignment.new(@experiment, 'subject', @experiment.group(:all), nil)
+    @event_logger.log_assignment(assignment)
+  end
+
+  def test_log_conversion
+    @logger.expects(:debug).with('[Verdict::Conversion] experiment=logger subject=subject goal=my_goal')
+    conversion = Verdict::Conversion.new(@experiment, 'subject', :my_goal)
+    @event_logger.log_conversion(conversion)
+  end
+end

data/test/experiment_test.rb

@@ -0,0 +1,280 @@
+require 'json'
+require 'test_helper'
+
+class ExperimentTest < MiniTest::Unit::TestCase
+
+  def test_no_qualifier
+    e = Verdict::Experiment.new('test')
+    assert !e.has_qualifier?
+    assert e.everybody_qualifies?
+  end
+
+  def test_qualifier
+    e = Verdict::Experiment.new('test') do |experiment|
+      qualify { |subject| subject.country == 'CA' }
+      groups do
+        group :all, 100
+      end
+    end
+
+    assert e.has_qualifier?
+    assert !e.everybody_qualifies?
+
+    subject_stub = Struct.new(:id, :country)
+    ca_subject = subject_stub.new(1, 'CA')
+    us_subject = subject_stub.new(1, 'US')
+
+    assert e.qualifier.call(ca_subject)
+    assert !e.qualifier.call(us_subject)
+
+    qualified = e.assign(ca_subject)
+    assert_kind_of Verdict::Assignment, qualified
+    assert_equal e.group(:all), qualified.group
+
+    non_qualified = e.assign(us_subject)
+    assert_kind_of Verdict::Assignment, non_qualified
+    assert !non_qualified.qualified?
+    assert_equal nil, non_qualified.group
+  end
+
+  def test_assignment
+    e = Verdict::Experiment.new('test') do
+      qualify { |subject| subject <= 2 }
+      groups do
+        group :a, :half
+        group :b, :rest
+      end
+    end
+
+    assignment = e.assign(1)
+    assert_kind_of Verdict::Assignment, assignment
+    assert assignment.qualified?
+    assert !assignment.returning?
+    assert_equal assignment.group, e.group(:a)
+
+    assignment = e.assign(3)
+    assert_kind_of Verdict::Assignment, assignment
+    assert !assignment.qualified?
+
+    assert_equal :a,  e.switch(1)
+    assert_equal :b,  e.switch(2)
+    assert_equal nil, e.switch(3)
+  end
+
+  def test_subject_identifier
+    e = Verdict::Experiment.new('test')
+    assert_equal '123', e.retrieve_subject_identifier(stub(id: 123, to_s: '456'))
+    assert_equal '456', e.retrieve_subject_identifier(stub(to_s: '456'))
+    assert_raises(Verdict::EmptySubjectIdentifier) { e.retrieve_subject_identifier(stub(id: nil)) }
+    assert_raises(Verdict::EmptySubjectIdentifier) { e.retrieve_subject_identifier(stub(to_s: '')) }
+  end
+
+  def test_new_unqualified_assignment_without_store_unqualified
+    mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
+    e = Verdict::Experiment.new('test') do
+      qualify { mock_qualifier.qualifies? }
+      storage mock_store, store_unqualified: false
+    end
+
+    mock_qualifier.expects(:qualifies?).returns(false)
+    mock_store.expects(:retrieve_assignment).never
+    mock_store.expects(:store_assignment).never
+    e.assign(mock('subject'))
+  end
+
+  def test_returning_qualified_assignment_without_store_unqualified
+    mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
+    e = Verdict::Experiment.new('test') do
+      qualify { mock_qualifier.qualifies? }
+      storage mock_store, store_unqualified: false
+      groups { group :all, 100 }
+    end
+
+    qualified_assignment = e.subject_assignment(mock('identifier'), e.group(:all), Time.now)
+    mock_qualifier.expects(:qualifies?).returns(true)
+    mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
+    mock_store.expects(:store_assignment).never
+    e.assign(mock('subject'))
+  end    
+
+  def test_new_unqualified_assignment_with_store_unqualified
+    mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
+    e = Verdict::Experiment.new('test') do
+      qualify { mock_qualifier.qualifies? }
+      storage mock_store, store_unqualified: true
+    end
+
+    mock_qualifier.expects(:qualifies?).returns(false)
+    mock_store.expects(:retrieve_assignment).returns(nil).once
+    mock_store.expects(:store_assignment).once
+    e.assign(mock('subject'))
+  end
+
+  def test_returning_unqualified_assignment_with_store_unqualified
+    mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
+    e = Verdict::Experiment.new('test') do
+      qualify { mock_qualifier.qualifies? }
+      storage mock_store, store_unqualified: true
+    end
+
+    unqualified_assignment = e.subject_assignment(mock('subject_identifier'), nil, Time.now)
+    mock_qualifier.expects(:qualifies?).never
+    mock_store.expects(:retrieve_assignment).returns(unqualified_assignment).once
+    mock_store.expects(:store_assignment).never
+    e.assign(mock('subject'))
+  end
+
+  def test_returning_qualified_assignment_with_store_unqualified
+    mock_store, mock_qualifier = Verdict::Storage::MockStorage.new, mock('qualifier')
+    e = Verdict::Experiment.new('test') do
+      qualify { mock_qualifier.qualifies? }
+      storage mock_store, store_unqualified: true
+      groups { group :all, 100 }
+    end
+
+    qualified_assignment = e.subject_assignment(mock('subject_identifier'), e.group(:all), Time.now)
+    mock_qualifier.expects(:qualifies?).never
+    mock_store.expects(:retrieve_assignment).returns(qualified_assignment).once
+    mock_store.expects(:store_assignment).never
+    e.assign(mock('subject'))
+  end
+
+  def test_disqualify
+    e = Verdict::Experiment.new('test') do
+      groups { group :all, 100 }
+    end
+
+    subject = stub(id: 'walrus')
+    original_assignment = e.assign(subject)
+    assert original_assignment.qualified?
+    new_assignment = e.disqualify(subject)
+    assert !new_assignment.qualified?
+  end
+
+  def test_assignment_event_logging
+    e = Verdict::Experiment.new('test') do
+      groups { group :all, 100 }
+    end
+
+    e.stubs(:event_logger).returns(logger = mock('event_logger'))
+    logger.expects(:log_assignment).with(kind_of(Verdict::Assignment))
+
+    e.assign(stub(id: 'subject_identifier'))
+  end
+
+  def test_conversion_event_logging
+    e = Verdict::Experiment.new('test')
+
+    e.stubs(:event_logger).returns(logger = mock('logger'))
+    logger.expects(:log_conversion).with(kind_of(Verdict::Conversion))
+
+    conversion = e.convert(subject = stub(id: 'test_subject'), :my_goal)
+    assert_equal 'test_subject', conversion.subject_identifier
+    assert_equal :my_goal, conversion.goal 
+  end
+
+  def test_json
+    e = Verdict::Experiment.new(:json) do
+      name 'testing'
+      subject_type 'visitor'
+      groups do
+        group :a, :half
+        group :b, :rest
+      end
+    end
+
+    Timecop.freeze(Time.new(2013, 2, 3, 4, 5, 6, '+00:00')) do
+      e.send(:ensure_experiment_has_started)
+    end
+
+    json = JSON.parse(e.to_json)
+    assert_equal 'json', json['handle']
+    assert_equal false, json['has_qualifier']
+    assert_kind_of Enumerable, json['groups']
+    assert_equal 'testing', json['metadata']['name']
+    assert_equal 'visitor', json['subject_type']
+    assert_equal '2013-02-03T04:05:06Z', json['started_at']
+  end
+
+  def test_storage_read_failure
+    storage_mock = Verdict::Storage::MockStorage.new
+    e = Verdict::Experiment.new(:json) do
+      groups { group :all, 100 }
+      storage storage_mock
+    end
+
+    storage_mock.stubs(:retrieve_assignment).raises(Verdict::StorageError, 'storage read issues')
+    rescued_assignment = e.assign(stub(id: 123))
+    assert !rescued_assignment.qualified?
+  end
+
+  def test_storage_write_failure
+    storage_mock = Verdict::Storage::MockStorage.new
+    e = Verdict::Experiment.new(:json) do
+      groups { group :all, 100 }
+      storage storage_mock
+    end
+
+    storage_mock.expects(:retrieve_assignment).returns(e.subject_assignment(mock('subject_identifier'), e.group(:all)))
+    storage_mock.expects(:store_assignment).raises(Verdict::StorageError, 'storage write issues')
+    rescued_assignment = e.assign(stub(id: 456))
+    assert !rescued_assignment.qualified?
+  end
+
+  def test_initial_started_at
+    e = Verdict::Experiment.new('test') do
+      groups { group :all, 100 }
+    end
+
+    e.subject_storage.expects(:retrieve_start_timestamp).returns(nil)
+    e.subject_storage.expects(:store_start_timestamp).once
+    e.send(:ensure_experiment_has_started)
+  end
+
+  def test_subsequent_started_at_when_start_time_is_memoized
+    e = Verdict::Experiment.new('test') do
+      groups { group :all, 100 }
+    end
+
+    e.send(:ensure_experiment_has_started)
+    e.subject_storage.expects(:retrieve_start_timestamp).never
+    e.subject_storage.expects(:store_start_timestamp).never
+    e.send(:ensure_experiment_has_started)
+  end
+
+  def test_subsequent_started_at_when_start_time_is_not_memoized
+    e = Verdict::Experiment.new('test') do
+      groups { group :all, 100 }
+    end
+
+    e.subject_storage.expects(:retrieve_start_timestamp).returns(Time.now.utc)
+    e.subject_storage.expects(:store_start_timestamp).never
+    e.send(:ensure_experiment_has_started)
+  end
+
+  def test_qualify_based_on_experiment_start_timestamp
+    Timecop.freeze(Time.new(2012)) do
+      e = Verdict::Experiment.new('test') do
+        qualify { |subject| subject.created_at >= self.started_at }
+        groups { group :all, 100 }
+      end
+
+      subject = stub(id: 'old', created_at: Time.new(2011))
+      assert !e.assign(subject).qualified?
+
+      subject = stub(id: 'new', created_at: Time.new(2013))
+      assert e.assign(subject).qualified?
+    end
+  end
+
+  def test_experiment_starting_behavior
+    e = Verdict::Experiment.new('starting_test') do
+      groups { group :all, 100 }
+    end
+
+    assert !e.started?, "The experiment should not have started yet"
+
+    e.assign(stub(id: '123'))
+    assert e.started?, "The experiment should have started after the first assignment"
+  end
+end

data/test/experiments_repository_test.rb

@@ -0,0 +1,32 @@
+require 'test_helper'
+
+class ExperimentTest < MiniTest::Unit::TestCase
+
+  def setup
+    Verdict.repository.clear
+  end
+
+  def test_should_keep_list_of_all_experiments
+    size_at_start = Verdict.repository.size
+    e = Verdict::Experiment.define('test')
+
+    assert_equal size_at_start + 1, Verdict.repository.size
+    assert_equal e, Verdict['test']
+  end
+
+  def test_should_not_allow_experiments_with_the_same_name
+    Verdict::Experiment.define('test_duplicate')
+    assert_raises(Verdict::ExperimentHandleNotUnique) do
+      Verdict::Experiment.define('test_duplicate')
+    end
+  end
+
+  def test_json_export
+    e1 = Verdict::Experiment.define('test_1') { groups { group :all, 100 } }
+    e2 = Verdict::Experiment.define('test_2') { groups { group :all, 100 } }
+
+    json = JSON.parse(Verdict.repository.to_json)
+    assert_equal ['test_1', 'test_2'], json.keys
+    assert_equal json['test_1'], JSON.parse(e1.to_json)
+  end
+end

data/test/group_test.rb

@@ -0,0 +1,42 @@
+require 'test_helper'
+
+class GroupTest < MiniTest::Unit::TestCase
+
+  def setup
+    @experiment = Verdict::Experiment.new('a')
+  end
+
+  def test_basic_properties
+    group = Verdict::Group.new(@experiment, :test)
+
+    assert_equal @experiment, group.experiment
+    assert_kind_of Verdict::Group, group
+    assert_equal 'test', group.handle
+    assert_equal 'test', group.to_s
+    assert_equal :test, group.to_sym
+  end
+
+  def test_triple_equals
+    group = Verdict::Group.new(@experiment, 'control')
+    assert group === Verdict::Group.new(@experiment, :control)
+    assert group === 'control'
+    assert group === :control
+    assert !(group === nil)
+
+    assert !(group === Verdict::Group.new(@experiment, :test))
+    assert !(group === Verdict::Group.new(Verdict::Experiment.new('b'), :test))
+    assert !(group === 'test')
+    assert !(group === nil)
+  end
+
+  def test_json
+    group = Verdict::Group.new(@experiment, 'control')
+    group.name 'testing'
+    group.description 'description'
+
+    json = JSON.parse(group.to_json)
+    assert_equal 'control', json['handle']
+    assert_equal 'testing', json['metadata']['name']
+    assert_equal 'description', json['metadata']['description']
+  end  
+end

data/test/memory_subject_storage_test.rb

@@ -0,0 +1,45 @@
+require 'test_helper'
+
+class MemorySubjectStorageTest < MiniTest::Unit::TestCase
+
+  def setup
+    @storage = storage = Verdict::Storage::MemoryStorage.new
+    @experiment = Verdict::Experiment.new(:memory_storage) do
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+
+    @subject = stub(id: 'bootscale')
+  end
+
+  def test_wrapup
+    @experiment.assign(@subject)
+    @experiment.wrapup
+    assert @experiment.lookup(@subject).nil?
+  end
+
+  def test_with_memory_store
+    assignment_1 = @experiment.assign(@subject)
+    assignment_2 = @experiment.assign(@subject)
+    assert !assignment_1.returning?
+    assert assignment_2.returning?
+  end
+
+  def test_assignment_lookup
+    assert @experiment.lookup(@subject).nil?
+    @experiment.assign(@subject)
+    assert !@experiment.lookup(@subject).nil?
+  end
+
+  def test_remove_assignment
+    assert !@experiment.assign(@subject).returning?
+    @experiment.wrapup
+    assert !@experiment.assign(@subject).returning?
+  end
+
+  def test_started_at
+    assert @storage.start_timestamps[@experiment.handle].nil?
+    @experiment.send(:ensure_experiment_has_started)
+    assert @storage.start_timestamps[@experiment.handle].instance_of?(Time)
+  end
+end

data/test/metadata_test.rb

@@ -0,0 +1,44 @@
+require 'test_helper'
+
+class MetadataTest < MiniTest::Unit::TestCase
+
+  def test_experiment_metadata
+    experiment = Verdict::Experiment.new('experiment metadata') do
+      name "Metadata test"
+      description "For testing metadata functionality"
+      owner "Willem van Bergen"
+    end
+
+    assert_equal "Metadata test", experiment.name
+    assert_equal "For testing metadata functionality", experiment.description
+    assert_equal "Willem van Bergen", experiment.owner
+
+    assert_equal experiment.metadata, {
+      :name => 'Metadata test',
+      :description => 'For testing metadata functionality',
+      :owner => 'Willem van Bergen'
+    }
+  end
+
+  def test_group_metadata
+    experiment = Verdict::Experiment.new('group metadata') do
+      groups do
+        group :all, 100 do
+          name "Group metadata test"
+          description "For testing metadata functionality"
+          screenshot "http://example.com/image.png"
+        end
+      end
+    end
+
+    assert_equal "Group metadata test", experiment.group(:all).name
+    assert_equal "For testing metadata functionality", experiment.group(:all).description
+    assert_equal "http://example.com/image.png", experiment.group(:all).screenshot
+
+    assert_equal experiment.group(:all).metadata, {
+      :name => 'Group metadata test',
+      :description => 'For testing metadata functionality',
+      :screenshot => 'http://example.com/image.png'
+    }
+  end
+end