verdict 0.13.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec6df6bdd38dc947ff1072e1dce844ab4d3ddd522701c8de578a35b4c9c51be8
-  data.tar.gz: e6b62b2c579a363f37526079389bea4cb16eb28ed7573dec2d3fa8f8c74ee4ed
+  metadata.gz: c8ad702f655cca4734df9f1019f7df665cbe04914468bdcb07ae2a5e9fcc27ca
+  data.tar.gz: f31057b426bb631a36d99bc9a6f1588078b0009909ad25c2f5daa2180a407d66
 SHA512:
-  metadata.gz: 1399fa37c4575888ffb83aea54469784db11ceef1ebc99f5c1fa5de75b936f460b1f34dc717c28f4f19a6e969c3a85a3faad022119113cd94f754ad61b4fdb85
-  data.tar.gz: 7f1836b8d23ca79a4338f0a4f4ed244ebe7bdbd8b9f03529f2cb00e110c280b1033e112928da900d805112eb0036e0f74278ea021514016541accf01d8c3308c
+  metadata.gz: 1ac4f9f4cef8e978fc36ea7848cee6b4ab15cb895d57598388f4b42997f141189a51c511f4a4b877f254b8a0d32909a153aa00da8b7e42940dbaed6d7ec9e4ec
+  data.tar.gz: f03f49ca42347dc3bf54c0ab449df49df32e6b5fc175392e3259b4e238324de03406f64d38a17e8744f662ec9543319a41bebec0df12031f9f0ab03bb05253c2

data/.travis.yml

@@ -3,9 +3,9 @@ cache: bundler
 before_install: gem update bundler
 script: bundle exec rake
 rvm:
-  - 2.0
-  - 2.1
-  - 2.3.3
+  - 2.5
+  - 2.6
+  - 2.7
   - ruby-head
   - jruby
 matrix:

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## v0.16.0
+* Allow configuring the `RedisStorage` with a [`ConnectionPool`](https://github.com/mperham/connection_pool) instead of a raw `Redis` connection.
+
+## v0.15.2
+* Fix edge case where inheriting from `Verdict::Experiment` and overriding `subject_qualifies?` resulted in an error.
+
+## v0.15.1
+* Make the `dynamic_qualifies` parameter in `Verdict::Experiment#subject_qualifies?` optional. This fixes a bug where users that were previously calling this method directly experienced issues after v0.15.0
+
+## v0.15.0
+* Add optional `qualifiers` parameter to the `Verdict::Experiment#switch` method. This parameter accepts an array of procs and is used as additional qualifiers. The purpose of this parameter is to allow users to define qualification logic outside of the experiment definition.
+
+## v0.14.0
+* Add optional experiment definition method `schedule_stop_new_assignment_timestamp` to support limiting experiment's assignment lifetime with another pre-determined time interval. It allows users to have an assignment cooldown period for stable analysis of the experiment results. Experiment's lifetime now becomes: start experiment -> stop new assignments -> end experiment. 
+
 ## v0.13.0
 
 * Add optional experiment definition methods `schedule_start_timestamp` and `schedule_end_timestamp` to support limiting experiment's lifetime in a pre-determined time interval.

data/lib/verdict/experiment.rb

@@ -52,9 +52,17 @@ class Verdict::Experiment
     return self
   end
 
-  # Optional: Together with the "end timestamp", limits the experiment run timeline within
-  # the given time interval. When experiment is not scheduled, subject switch returns nil.
-  # This is useful when the experimenter requires the experiment to run in a strict timeline.
+  # Optional: Together with the "end_timestamp" and "stop_new_assignment_timestamp", limits the experiment run timeline within
+  # the given time interval.
+  #
+  # Timestamps definitions:
+  # start_timestamp: Experiment's start time. No assignments are made i.e. switch will return nil before this timestamp.
+  # stop_new_assignment_timestamp: Experiment's new assignment stop time. No new assignments are made
+  # i.e. switch returns nil for new assignments but the existing assignments are preserved.
+  # end_timestamp: Experiment's end time. No assignments are made i.e. switch returns nil after this timestamp.
+  #
+  # Experiment run timeline:
+  # start_timestamp -> (new assignments occur) -> stop_new_assignment_timestamp -> (no new assignments occur) -> end_timestamp
   def schedule_start_timestamp(timestamp)
     @schedule_start_timestamp = timestamp
   end
@@ -63,6 +71,10 @@ class Verdict::Experiment
     @schedule_end_timestamp = timestamp
   end
 
+  def schedule_stop_new_assignment_timestamp(timestamp)
+    @schedule_stop_new_assignment_timestamp = timestamp
+  end
+
   def rollout_percentage(percentage, rollout_group_name = :enabled)
     groups(Verdict::Segmenters::RolloutSegmenter) do
       group rollout_group_name, percentage
@@ -130,18 +142,18 @@ class Verdict::Experiment
     raise unless disqualify_empty_identifier?
   end
 
-  def assign(subject, context = nil)
+  def assign(subject, context = nil, dynamic_qualifiers: [])
     previous_assignment = lookup(subject)
 
     subject_identifier = retrieve_subject_identifier(subject)
     assignment = if previous_assignment
-                   previous_assignment
-                 elsif subject_qualifies?(subject, context)
-                   group = segmenter.assign(subject_identifier, subject, context)
-                   subject_assignment(subject, group, nil, group.nil?)
-                 else
-                   nil_assignment(subject)
-                 end
+      previous_assignment
+    elsif dynamic_subject_qualifies?(subject, dynamic_qualifiers, context) && is_make_new_assignments?
+      group = segmenter.assign(subject_identifier, subject, context)
+      subject_assignment(subject, group, nil, group.nil?)
+    else
+      nil_assignment(subject)
+    end
 
     store_assignment(assignment)
   rescue Verdict::StorageError
@@ -182,9 +194,11 @@ class Verdict::Experiment
     @storage.remove_assignment(self, subject)
   end
 
-  def switch(subject, context = nil)
+  # The qualifiers param accepts an array of procs.
+  # This is intended for qualification logic that cannot be defined in the experiment definition
+  def switch(subject, context = nil, qualifiers: [])
     return unless is_scheduled?
-    assign(subject, context).to_sym
+    assign(subject, context, dynamic_qualifiers: qualifiers).to_sym
   end
 
   def lookup(subject)
@@ -229,8 +243,9 @@ class Verdict::Experiment
     @disqualify_empty_identifier
   end
 
-  def subject_qualifies?(subject, context = nil)
+  def subject_qualifies?(subject, context = nil, dynamic_qualifiers: [])
     ensure_experiment_has_started
+    return false unless dynamic_qualifiers.all? { |qualifier| qualifier.call(subject) }
     everybody_qualifies? || @qualifiers.all? { |qualifier| qualifier.call(subject, context) }
   end
 
@@ -268,12 +283,27 @@ class Verdict::Experiment
   private
 
   def is_scheduled?
-    if @schedule_start_timestamp and @schedule_start_timestamp > Time.now
+    if @schedule_start_timestamp && @schedule_start_timestamp > Time.now
       return false
     end
-    if @schedule_end_timestamp and @schedule_end_timestamp < Time.now
+    if @schedule_end_timestamp && @schedule_end_timestamp <= Time.now
       return false
     end
     return true
   end
+
+  def is_make_new_assignments?
+    return !(@schedule_stop_new_assignment_timestamp && @schedule_stop_new_assignment_timestamp <= Time.now)
+  end
+
+  # Used when a Experiment class has overridden the subject_qualifies? method prior to v0.15.0
+  # The previous version of subject_qualifies did not accept dynamic qualifiers, thus this is used to
+  # determine how many parameters to pass
+  def dynamic_subject_qualifies?(subject, dynamic_qualifiers, context)
+    if method(:subject_qualifies?).parameters.include?([:key, :dynamic_qualifiers])
+      subject_qualifies?(subject, context, dynamic_qualifiers: dynamic_qualifiers)
+    else
+      subject_qualifies?(subject, context)
+    end
+  end
 end

data/lib/verdict/storage/redis_storage.rb

@@ -6,33 +6,56 @@ module Verdict
       attr_accessor :redis, :key_prefix
 
       def initialize(redis = nil, options = {})
-        @redis = redis
+        if !redis.nil? && !redis.respond_to?(:with)
+          @redis = ConnectionPoolLike.new(redis)
+        else
+          @redis = redis
+        end
+
         @key_prefix = options[:key_prefix] || 'experiments/'
       end
 
       protected
 
+      class ConnectionPoolLike
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def with
+          yield @redis
+        end
+      end
+
       def get(scope, key)
-        redis.hget(scope_key(scope), key)
+        redis.with do |conn|
+          conn.hget(scope_key(scope), key)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def set(scope, key, value)
-        redis.hset(scope_key(scope), key, value)
+        redis.with do |conn|
+          conn.hset(scope_key(scope), key, value)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def remove(scope, key)
-        redis.hdel(scope_key(scope), key)
+        redis.with do |conn|
+          conn.hdel(scope_key(scope), key)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def clear(scope, options)
         scrub(scope)
-        redis.del(scope_key(scope))
+        redis.with do |conn|
+          conn.del(scope_key(scope))
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
@@ -44,10 +67,14 @@ module Verdict
       end
 
       def scrub(scope, cursor: 0)
-        cursor, results = redis.hscan(scope_key(scope), cursor, count: PAGE_SIZE)
+        cursor, results = redis.with do |conn|
+          conn.hscan(scope_key(scope), cursor, count: PAGE_SIZE)
+        end
+
         results.map(&:first).each do |key|
           remove(scope, key)
         end
+
         scrub(scope, cursor: cursor) unless cursor.to_i.zero?
       end
     end

data/lib/verdict/version.rb

@@ -1,3 +1,3 @@
 module Verdict
-  VERSION = "0.13.0"
+  VERSION = "0.16.0"
 end

data/test/experiment_test.rb

@@ -565,9 +565,134 @@ class ExperimentTest < Minitest::Test
     end
   end
 
+  def test_is_stop_new_assignments
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+      schedule_stop_new_assignment_timestamp Time.new(2020, 1, 15)
+    end
+
+    # new assignments stopped after the stop timestamp
+    Timecop.freeze(Time.new(2020, 1, 16)) do
+      assert !e.send(:is_make_new_assignments?)
+      assert_nil e.switch(1)
+    end
+    # new assignments didn't stop before the stop timestamp
+    Timecop.freeze(Time.new(2020, 1, 3)) do
+      assert e.send(:is_make_new_assignments?)
+      assert :a, e.switch(2)
+    end
+  end
+
+  def test_switch_preserves_old_assignments_after_stop_new_assignments_timestamp
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+    end
+
+    assert_equal :a, e.switch(1)
+
+    e.schedule_stop_new_assignment_timestamp Time.new(2020, 4, 15)
+
+    # switch respects to stop new assignment timestamp, old assignment preserves, new assignment returns nil
+    Timecop.freeze(Time.new(2020, 4, 16)) do
+      assert !e.send(:is_make_new_assignments?)
+      # old assignment stay the same
+      assert_equal :a, e.switch(1)
+      # new assignment returns nil
+      assert_nil e.switch(2)
+    end
+  end
+
+  def test_schedule_start_timestamp_and_stop_new_assignemnt_timestamp_are_inclusive_but_end_timestamp_is_exclusive
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+
+      schedule_start_timestamp Time.new(2020, 1, 1)
+      schedule_stop_new_assignment_timestamp Time.new(2020, 1, 15)
+      schedule_end_timestamp Time.new(2020, 1, 31)
+    end
+
+    # start_timestamp is included
+    Timecop.freeze(Time.new(2020, 1, 1)) do
+      assert e.send(:is_scheduled?)
+      assert_equal :a, e.switch(1)
+    end
+
+    # stop_new_assignment_timestamp is included
+    Timecop.freeze(Time.new(2020, 1, 15)) do
+      assert !e.send(:is_make_new_assignments?)
+      # old assignment preserved
+      assert_equal :a, e.switch(1)
+      # new assignment returns nil
+      assert_nil e.switch(2)
+    end
+
+    # end_timestamp is excluded
+    Timecop.freeze(Time.new(2020, 1, 31)) do
+      assert !e.send(:is_scheduled?)
+      assert_nil e.switch(1)
+    end
+  end
+
+  def test_custom_qualifiers_success
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    subject = 2
+    custom_qualifier_a = Proc.new { |subject| subject.even? }
+    custom_qualifier_b = Proc.new { |subject| subject > 0 }
+
+    group = e.switch(subject, qualifiers: [custom_qualifier_a, custom_qualifier_b])
+    assert_equal e.group(:all).to_sym, group
+  end
+
+  def test_custom_qualifiers_failure
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    subject = 3
+    custom_qualifier_a = Proc.new { |subject| subject.even? }
+    custom_qualifier_b = Proc.new { |subject| subject > 0 }
+
+    group = e.switch(subject, qualifiers: [custom_qualifier_a, custom_qualifier_b])
+    assert_nil group
+  end
+
+  def test_dynamic_subject_qualifies_call_overridden_method
+    e = MyExperiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    group = e.switch(4)
+    assert_nil group
+  end
+
   private
 
   def redis
     @redis ||= ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
   end
 end
+
+class MyExperiment < Verdict::Experiment
+  def subject_qualifies?(subject, context = nil)
+    return false if subject.even?
+    super
+  end
+end

data/test/storage/redis_storage_test.rb

@@ -1,97 +1,75 @@
 require 'test_helper'
 
-class RedisStorageTest < Minitest::Test
 
-  def setup
-    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
-    @storage = storage = Verdict::Storage::RedisStorage.new(@redis)
-    @experiment = Verdict::Experiment.new(:redis_storage) do
-      qualify { |s| s == 'subject_1' }
-      groups { group :all, 100 }
-      storage storage, store_unqualified: true
-    end
-  end
-
-  def teardown
-    @redis.del('experiments/redis_storage')
-  end
+module SharedRedisStorageTests
+  attr_reader :redis, :experiment, :storage
 
   def test_store_and_retrieve_qualified_assignment
-    refute @redis.hexists(experiment_key, 'assignment_subject_1')
+    refute redis.hexists(experiment_key, 'assignment_subject_1')
 
-    new_assignment = @experiment.assign('subject_1')
+    new_assignment = experiment.assign('subject_1')
     assert new_assignment.qualified?
     refute new_assignment.returning?
 
-    assert @redis.hexists(experiment_key, 'assignment_subject_1')
+    assert redis.hexists(experiment_key, 'assignment_subject_1')
 
-    returning_assignment = @experiment.assign('subject_1')
+    returning_assignment = experiment.assign('subject_1')
     assert returning_assignment.returning?
     assert_equal new_assignment.experiment, returning_assignment.experiment
     assert_equal new_assignment.group, returning_assignment.group
   end
 
   def test_store_and_retrieve_unqualified_assignment
-    refute @redis.hexists(experiment_key, 'assignment_subject_2')
+    refute redis.hexists(experiment_key, 'assignment_subject_2')
 
-    new_assignment = @experiment.assign('subject_2')
+    new_assignment = experiment.assign('subject_2')
 
     refute new_assignment.returning?
     refute new_assignment.qualified?
-    assert @redis.hexists(experiment_key, 'assignment_subject_2')
+    assert redis.hexists(experiment_key, 'assignment_subject_2')
 
-    returning_assignment = @experiment.assign('subject_2')
+    returning_assignment = experiment.assign('subject_2')
     assert returning_assignment.returning?
     assert_equal new_assignment.experiment, returning_assignment.experiment
     assert_nil new_assignment.group
     assert_nil returning_assignment.group
   end
 
-  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_reads
-    @redis.stubs(:hget).raises(::Redis::BaseError, "Redis is down")
-    assert !@experiment.assign('subject_1').qualified?
-  end
-
-  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
-    @redis.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
-    assert !@experiment.assign('subject_1').qualified?
-  end
-
   def test_remove_subject_assignment
-    @experiment.assign('subject_3')
-    assert @redis.hexists(experiment_key, 'assignment_subject_3')
-    @experiment.remove_subject_assignment('subject_3')
-    refute @redis.hexists(experiment_key, 'assignment_subject_3')
+    experiment.assign('subject_3')
+    assert redis.hexists(experiment_key, 'assignment_subject_3')
+    experiment.remove_subject_assignment('subject_3')
+    refute redis.hexists(experiment_key, 'assignment_subject_3')
   end
 
   def test_started_at
-    refute @redis.hexists(experiment_key, "started_at")
-    a = @experiment.send(:ensure_experiment_has_started)
-    assert @redis.hexists(experiment_key, "started_at")
-    assert_equal a, @experiment.started_at
+    refute redis.hexists(experiment_key, "started_at")
+    a = experiment.send(:ensure_experiment_has_started)
+    assert redis.hexists(experiment_key, "started_at")
+    assert_equal a, experiment.started_at
   end
 
   def test_cleanup_with_scope_argument
     1000.times do |n|
-      @experiment.assign("something_#{n}")
+      experiment.assign("something_#{n}")
     end
 
-    assert_operator @redis, :exists, experiment_key
+    assert_operator redis, :exists, experiment_key
 
     Verdict.default_logger.expects(:warn).with(regexp_matches(/deprecated/))
-    @storage.cleanup(:redis_storage)
-    refute_operator @redis, :exists, experiment_key
+    storage.cleanup(:redis_storage)
+    refute_operator redis, :exists, experiment_key
   end
 
   def test_cleanup
     1000.times do |n|
-      @experiment.assign("something_#{n}")
+      experiment.assign("something_#{n}")
     end
 
-    assert_operator @redis, :exists, experiment_key
+    assert_operator redis, :exists, experiment_key
 
-    @storage.cleanup(@experiment)
-    refute_operator @redis, :exists, experiment_key
+    storage.cleanup(experiment)
+    refute_operator redis, :exists, experiment_key
   end
 
   private
@@ -100,3 +78,67 @@ class RedisStorageTest < Minitest::Test
     "experiments/redis_storage"
   end
 end
+
+class RedisStorageTest < Minitest::Test
+  include SharedRedisStorageTests
+
+  def setup
+    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    @storage = storage = Verdict::Storage::RedisStorage.new(@redis)
+    @experiment = Verdict::Experiment.new(:redis_storage) do
+      qualify { |s| s == 'subject_1' }
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+  end
+
+  def teardown
+    @redis.del('experiments/redis_storage')
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_reads
+    redis.stubs(:hget).raises(::Redis::BaseError, "Redis is down")
+    assert !experiment.assign('subject_1').qualified?
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
+    redis.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
+    assert !experiment.assign('subject_1').qualified?
+  end
+end
+
+class RedisStorageConnectionPoolTest < Minitest::Test
+  include SharedRedisStorageTests
+
+  def setup
+    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    @redis_pool = ::ConnectionPool.new(size: 3) do
+      ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    end
+    @storage = storage = Verdict::Storage::RedisStorage.new(@redis_pool)
+    @experiment = Verdict::Experiment.new(:redis_storage) do
+      qualify { |s| s == 'subject_1' }
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+  end
+
+  def teardown
+    @redis.del('experiments/redis_storage')
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
+    redis_mock = stub
+    redis_mock.stubs(:hget).returns(nil)
+    redis_mock.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
+    @redis_pool.stubs(:with).yields(redis_mock)
+    assert !@experiment.assign('subject_1').qualified?
+  end
+
+  def test_remove_subject_assignment
+    @experiment.assign('subject_3')
+    assert @redis.hexists(experiment_key, 'assignment_subject_3')
+    @experiment.remove_subject_assignment('subject_3')
+    refute @redis.hexists(experiment_key, 'assignment_subject_3')
+  end
+end

data/test/test_helper.rb

@@ -13,6 +13,7 @@ require "mocha/setup"
 require "timecop"
 require "verdict"
 require "redis"
+require "connection_pool"
 
 REDIS_HOST = ENV['REDIS_HOST'].nil? ? '127.0.0.1' : ENV['REDIS_HOST']
 REDIS_PORT = (ENV['REDIS_PORT'].nil? ? '6379' : ENV['REDIS_PORT']).to_i

data/verdict.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("timecop")
   gem.add_development_dependency("redis")
   gem.add_development_dependency("rails")
+  gem.add_development_dependency("connection_pool")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: verdict
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-03 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Shopify Experiments classes
 email:
 - kevin.mcphillips@shopify.com