verdict 0.1.0

data/test/redis_subject_storage_test.rb

@@ -0,0 +1,89 @@
+require 'test_helper'
+
+class RedisSubjectStorageTest < MiniTest::Unit::TestCase
+
+  def setup
+    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    @storage = storage = Verdict::Storage::RedisStorage.new(@redis)
+    @experiment = Verdict::Experiment.new(:redis_storage) do
+      qualify { |s| s == 'subject_1' }
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+  end
+
+  def teardown
+    @storage.clear_experiment(@experiment)
+  end
+
+  def test_generate_experiment_key_should_generate_namespaced_key
+    assert_equal 'experiments/redis_storage', @storage.send(:generate_experiment_key, @experiment)
+  end
+
+  def test_store_and_retrieve_qualified_assignment
+    experiment_key = @storage.send(:generate_experiment_key, @experiment)
+    assert !@redis.hexists(experiment_key, 'subject_1')
+
+    new_assignment = @experiment.assign('subject_1')
+    assert new_assignment.qualified?
+    assert !new_assignment.returning?
+
+    assert @redis.hexists(experiment_key, 'subject_1')
+
+    returning_assignment = @experiment.assign('subject_1')
+    assert returning_assignment.returning?
+    assert_equal new_assignment.experiment, returning_assignment.experiment
+    assert_equal new_assignment.group, returning_assignment.group
+  end
+
+  def test_store_and_retrieve_unqualified_assignment
+    experiment_key = @storage.send(:generate_experiment_key, @experiment)
+    assert !@redis.hexists(experiment_key, 'subject_2')
+
+    new_assignment = @experiment.assign('subject_2')
+
+    assert !new_assignment.returning?
+    assert !new_assignment.qualified?
+    assert @redis.hexists(experiment_key, 'subject_2')
+
+    returning_assignment = @experiment.assign('subject_2')
+    assert returning_assignment.returning?
+    assert_equal new_assignment.experiment, returning_assignment.experiment
+    assert_equal new_assignment.group, returning_assignment.group
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_reads
+    @redis.stubs(:hget).raises(::Redis::BaseError, "Redis is down")
+    assert !@experiment.assign('subject_1').qualified?
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
+    @redis.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
+    assert !@experiment.assign('subject_1').qualified?
+  end
+
+  def test_remove_assignment
+    experiment_key = @storage.send(:generate_experiment_key, @experiment)
+    @experiment.assign('subject_3')
+    assert @redis.hexists(experiment_key, 'subject_3')
+    @experiment.remove_subject('subject_3')
+    assert !@redis.hexists(experiment_key, 'subject_3')
+  end
+
+  def test_clear_experiment
+    experiment_key = @storage.send(:generate_experiment_key, @experiment)
+    new_assignment = @experiment.assign('subject_3')
+    assert @redis.exists(experiment_key)
+    @experiment.wrapup
+    assert !@redis.exists(experiment_key)
+  end
+
+  def test_started_at
+    key = @storage.send(:generate_experiment_start_timestamp_key, @experiment)
+    
+    assert !@redis.exists(key)
+    a = @experiment.send(:ensure_experiment_has_started)
+    assert @redis.exists(key)
+    assert_equal a, @experiment.started_at
+  end
+end

data/test/static_percentage_segmenter_test.rb

@@ -0,0 +1,94 @@
+require 'test_helper'
+
+class StaticPercentageSegmenterTest < MiniTest::Unit::TestCase
+
+  MockExperiment = Struct.new(:handle)
+
+  def setup
+    Verdict.repository.clear
+  end
+
+  def test_add_up_to_100_percent
+    s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+    s.group :segment1, 1
+    s.group :segment2, 54
+    s.group :segment3, 27
+    s.group :segment4, 18
+    s.verify!
+
+    assert_equal ['segment1', 'segment2', 'segment3', 'segment4'], s.groups.keys
+    assert_equal  0 ...   1, s.groups['segment1'].percentile_range
+    assert_equal  1 ...  55, s.groups['segment2'].percentile_range
+    assert_equal 55 ...  82, s.groups['segment3'].percentile_range
+    assert_equal 82 ... 100, s.groups['segment4'].percentile_range
+  end
+
+  def test_definition_ofhalf_and_rest
+    s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+    s.group :first_half, :half
+    s.group :second_half, :rest
+    s.verify!
+
+    assert_equal ['first_half', 'second_half'], s.groups.keys
+    assert_equal  0 ...  50, s.groups['first_half'].percentile_range
+    assert_equal 50 ... 100, s.groups['second_half'].percentile_range
+  end
+
+  def test_raises_if_less_than_100_percent
+    assert_raises(Verdict::SegmentationError) do
+      s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+      s.group :too_little, 99
+      s.verify!
+    end
+  end
+
+  def test_raises_if_greather_than_100_percent
+    assert_raises(Verdict::SegmentationError) do
+      s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+      s.group :too_much, 101
+      s.verify!
+    end
+  end
+
+  def test_consistent_assignment_for_subjects
+    s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+    s.group :first_half, :half
+    s.group :second_half, :rest
+    s.verify!
+
+    3.times do 
+      assert s.groups['first_half']  === s.assign(1, nil, nil)
+      assert s.groups['second_half'] === s.assign(2, nil, nil)
+    end
+  end
+
+  def test_fair_segmenting
+    s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+    s.group :first_third, 33
+    s.group :second_third, 33
+    s.group :final_third, :rest
+    s.verify!
+
+    assignments = { :first_third => 0, :second_third => 0, :final_third => 0 }
+    200.times do |n| 
+      assignment = s.assign(n, nil, nil) 
+      assignments[assignment.to_sym] += 1
+    end
+
+    assert_equal 200, assignments.values.reduce(0, :+)
+    assert (60..72).include?(assignments[:first_third]),  'The groups should be roughly the same size.'
+    assert (60..72).include?(assignments[:second_third]), 'The groups should be roughly the same size.'
+    assert (60..72).include?(assignments[:final_third]),  'The groups should be roughly the same size.'
+  end
+
+  def test_group_json_export
+    s = Verdict::Segmenter::StaticPercentage.new(MockExperiment.new('test'))
+    s.group :first_third, 33
+    s.group :rest, :rest
+    s.verify!
+
+    json = JSON.parse(s.groups['rest'].to_json)
+    assert_equal 'rest', json['handle']
+    assert_equal 67, json['percentage']
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require "bundler/setup"
+require "minitest/autorun"
+require "minitest/pride"
+require "mocha/setup"
+require "timecop"
+require "verdict"
+require "redis"
+
+REDIS_HOST = ENV['REDIS_HOST'].nil? ? '127.0.0.1' : ENV['REDIS_HOST']
+REDIS_PORT = (ENV['REDIS_PORT'].nil? ? '6379' : ENV['REDIS_PORT']).to_i

data/verdict.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'verdict/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "verdict"
+  gem.version       = Verdict::VERSION
+  gem.authors       = ["Shopify"]
+  gem.email         = ["kevin.mcphillips@shopify.com", "willem@shopify.com"]
+  gem.description   = %q{Shopify Experiments classes}
+  gem.summary       = %q{A library to centrally define experiments for your application, and collect assignment information.}
+  gem.homepage      = "http://github.com/Shopify/verdict"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_development_dependency "minitest", '~> 4.2'
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "mocha"
+  gem.add_development_dependency "timecop"
+  gem.add_development_dependency "redis"
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: verdict
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shopify
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Shopify Experiments classes
+email:
+- kevin.mcphillips@shopify.com
+- willem@shopify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/verdict.rb
+- lib/verdict/assignment.rb
+- lib/verdict/conversion.rb
+- lib/verdict/event_logger.rb
+- lib/verdict/experiment.rb
+- lib/verdict/group.rb
+- lib/verdict/metadata.rb
+- lib/verdict/railtie.rb
+- lib/verdict/segmenter.rb
+- lib/verdict/storage.rb
+- lib/verdict/tasks.rake
+- lib/verdict/version.rb
+- test/assignment_test.rb
+- test/conversion_test.rb
+- test/event_logger_test.rb
+- test/experiment_test.rb
+- test/experiments_repository_test.rb
+- test/group_test.rb
+- test/memory_subject_storage_test.rb
+- test/metadata_test.rb
+- test/redis_subject_storage_test.rb
+- test/static_percentage_segmenter_test.rb
+- test/test_helper.rb
+- verdict.gemspec
+homepage: http://github.com/Shopify/verdict
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.4
+signing_key: 
+specification_version: 4
+summary: A library to centrally define experiments for your application, and collect
+  assignment information.
+test_files:
+- test/assignment_test.rb
+- test/conversion_test.rb
+- test/event_logger_test.rb
+- test/experiment_test.rb
+- test/experiments_repository_test.rb
+- test/group_test.rb
+- test/memory_subject_storage_test.rb
+- test/metadata_test.rb
+- test/redis_subject_storage_test.rb
+- test/static_percentage_segmenter_test.rb
+- test/test_helper.rb
+has_rdoc: