verdict 0.14.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f05448f375933cba0d01c10a4e302b684242297e779d96cd3bf45cff2f6fea74
-  data.tar.gz: d8782ab103377d758a550f36f16fa40cec2caa2662333a99d180aab637985126
+  metadata.gz: a6d20d61b4b74ae3cdc111f872f18c6bcedf9165bed983dfc489d826edad4e99
+  data.tar.gz: 5a23b9b6b678b0e7c7ccdd8c4df24f6029ee6f5c8a3ba16e9850147974b0a490
 SHA512:
-  metadata.gz: 14317430460d1fcb88748507d20fd8be9f0a271d434951c6deb4b89203eeb3a5675a7c88e5e1d79bd6404c275bc3a9a3b35558a09df51284c100613963c4ba51
-  data.tar.gz: 39ad70804de5c962785117525ccd668155efd228860735de13569316d22e6e3ed65e050842753c7283bc21c299ee237a00885686d6963a94e3130594f5c61b21
+  metadata.gz: 3c931d20970cb9a75a8f9780d6648493dbf6ca2a5a9eb61f79f35896063b603b36f611cc1e6175710075d0271afad9f73efd795b09eb55c63848dc67def3a68c
+  data.tar.gz: '0599822d437c5e7e512a0a75e64312aaf97008d0ae31b75bacab6bdc8361a0750b22f9e14f31d70c7b41df771c20267a2719e883a0394b161e1d4367901252d7'

data/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - 2.5
+          - 2.6
+          - 2.7
+          - head
+          - jruby
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    name: Tests Ruby ${{ matrix.ruby }}
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Run tests
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rake
+        env:
+          REDIS_HOST: localhost

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## v0.16.1
+* Change `RedisStorage` scrub to be iterative to avoid SystemStackError while cleaning big experiments
+
+## v0.16.0
+* Allow configuring the `RedisStorage` with a [`ConnectionPool`](https://github.com/mperham/connection_pool) instead of a raw `Redis` connection.
+
+## v0.15.2
+* Fix edge case where inheriting from `Verdict::Experiment` and overriding `subject_qualifies?` resulted in an error.
+
+## v0.15.1
+* Make the `dynamic_qualifies` parameter in `Verdict::Experiment#subject_qualifies?` optional. This fixes a bug where users that were previously calling this method directly experienced issues after v0.15.0
+
+## v0.15.0
+* Add optional `qualifiers` parameter to the `Verdict::Experiment#switch` method. This parameter accepts an array of procs and is used as additional qualifiers. The purpose of this parameter is to allow users to define qualification logic outside of the experiment definition.
+
 ## v0.14.0
 * Add optional experiment definition method `schedule_stop_new_assignment_timestamp` to support limiting experiment's assignment lifetime with another pre-determined time interval. It allows users to have an assignment cooldown period for stable analysis of the experiment results. Experiment's lifetime now becomes: start experiment -> stop new assignments -> end experiment. 
 

data/Gemfile

@@ -2,7 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in experiments.gemspec
 gemspec
-gem "rubysl", platform: :rbx
 gem "json", platform: :rbx
 
 group :development do

data/dev.yml

@@ -0,0 +1,11 @@
+name: verdict
+
+type: ruby
+
+up:
+  - ruby: 2.7.1
+  - bundler
+
+commands:
+  test:
+    run: rake test

data/lib/verdict/experiment.rb

@@ -142,18 +142,18 @@ class Verdict::Experiment
     raise unless disqualify_empty_identifier?
   end
 
-  def assign(subject, context = nil)
+  def assign(subject, context = nil, dynamic_qualifiers: [])
     previous_assignment = lookup(subject)
 
     subject_identifier = retrieve_subject_identifier(subject)
     assignment = if previous_assignment
-                   previous_assignment
-                 elsif subject_qualifies?(subject, context) && is_make_new_assignments?
-                   group = segmenter.assign(subject_identifier, subject, context)
-                   subject_assignment(subject, group, nil, group.nil?)
-                 else
-                   nil_assignment(subject)
-                 end
+      previous_assignment
+    elsif dynamic_subject_qualifies?(subject, dynamic_qualifiers, context) && is_make_new_assignments?
+      group = segmenter.assign(subject_identifier, subject, context)
+      subject_assignment(subject, group, nil, group.nil?)
+    else
+      nil_assignment(subject)
+    end
 
     store_assignment(assignment)
   rescue Verdict::StorageError
@@ -194,9 +194,11 @@ class Verdict::Experiment
     @storage.remove_assignment(self, subject)
   end
 
-  def switch(subject, context = nil)
+  # The qualifiers param accepts an array of procs.
+  # This is intended for qualification logic that cannot be defined in the experiment definition
+  def switch(subject, context = nil, qualifiers: [])
     return unless is_scheduled?
-    assign(subject, context).to_sym
+    assign(subject, context, dynamic_qualifiers: qualifiers).to_sym
   end
 
   def lookup(subject)
@@ -241,8 +243,9 @@ class Verdict::Experiment
     @disqualify_empty_identifier
   end
 
-  def subject_qualifies?(subject, context = nil)
+  def subject_qualifies?(subject, context = nil, dynamic_qualifiers: [])
     ensure_experiment_has_started
+    return false unless dynamic_qualifiers.all? { |qualifier| qualifier.call(subject) }
     everybody_qualifies? || @qualifiers.all? { |qualifier| qualifier.call(subject, context) }
   end
 
@@ -292,4 +295,15 @@ class Verdict::Experiment
   def is_make_new_assignments?
     return !(@schedule_stop_new_assignment_timestamp && @schedule_stop_new_assignment_timestamp <= Time.now)
   end
+
+  # Used when a Experiment class has overridden the subject_qualifies? method prior to v0.15.0
+  # The previous version of subject_qualifies did not accept dynamic qualifiers, thus this is used to
+  # determine how many parameters to pass
+  def dynamic_subject_qualifies?(subject, dynamic_qualifiers, context)
+    if method(:subject_qualifies?).parameters.include?([:key, :dynamic_qualifiers])
+      subject_qualifies?(subject, context, dynamic_qualifiers: dynamic_qualifiers)
+    else
+      subject_qualifies?(subject, context)
+    end
+  end
 end

data/lib/verdict/storage/redis_storage.rb

@@ -6,33 +6,56 @@ module Verdict
       attr_accessor :redis, :key_prefix
 
       def initialize(redis = nil, options = {})
-        @redis = redis
+        if !redis.nil? && !redis.respond_to?(:with)
+          @redis = ConnectionPoolLike.new(redis)
+        else
+          @redis = redis
+        end
+
         @key_prefix = options[:key_prefix] || 'experiments/'
       end
 
       protected
 
+      class ConnectionPoolLike
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def with
+          yield @redis
+        end
+      end
+
       def get(scope, key)
-        redis.hget(scope_key(scope), key)
+        redis.with do |conn|
+          conn.hget(scope_key(scope), key)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def set(scope, key, value)
-        redis.hset(scope_key(scope), key, value)
+        redis.with do |conn|
+          conn.hset(scope_key(scope), key, value)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def remove(scope, key)
-        redis.hdel(scope_key(scope), key)
+        redis.with do |conn|
+          conn.hdel(scope_key(scope), key)
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
 
       def clear(scope, options)
         scrub(scope)
-        redis.del(scope_key(scope))
+        redis.with do |conn|
+          conn.del(scope_key(scope))
+        end
       rescue ::Redis::BaseError => e
         raise Verdict::StorageError, "Redis error: #{e.message}"
       end
@@ -44,11 +67,17 @@ module Verdict
       end
 
       def scrub(scope, cursor: 0)
-        cursor, results = redis.hscan(scope_key(scope), cursor, count: PAGE_SIZE)
-        results.map(&:first).each do |key|
-          remove(scope, key)
+        loop do
+          cursor, results = redis.with do |conn|
+            conn.hscan(scope_key(scope), cursor, count: PAGE_SIZE)
+          end
+
+          results.map(&:first).each do |key|
+            remove(scope, key)
+          end
+
+          break if cursor.to_i.zero?
         end
-        scrub(scope, cursor: cursor) unless cursor.to_i.zero?
       end
     end
   end

data/lib/verdict/version.rb

@@ -1,3 +1,3 @@
 module Verdict
-  VERSION = "0.14.0"
+  VERSION = "0.16.1"
 end

data/test/experiment_test.rb

@@ -641,9 +641,58 @@ class ExperimentTest < Minitest::Test
       assert_nil e.switch(1)
     end
   end
+
+  def test_custom_qualifiers_success
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    subject = 2
+    custom_qualifier_a = Proc.new { |subject| subject.even? }
+    custom_qualifier_b = Proc.new { |subject| subject > 0 }
+
+    group = e.switch(subject, qualifiers: [custom_qualifier_a, custom_qualifier_b])
+    assert_equal e.group(:all).to_sym, group
+  end
+
+  def test_custom_qualifiers_failure
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    subject = 3
+    custom_qualifier_a = Proc.new { |subject| subject.even? }
+    custom_qualifier_b = Proc.new { |subject| subject > 0 }
+
+    group = e.switch(subject, qualifiers: [custom_qualifier_a, custom_qualifier_b])
+    assert_nil group
+  end
+
+  def test_dynamic_subject_qualifies_call_overridden_method
+    e = MyExperiment.new('test') do
+      groups do
+        group :all, 100
+      end
+    end
+
+    group = e.switch(4)
+    assert_nil group
+  end
+
   private
 
   def redis
     @redis ||= ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
   end
 end
+
+class MyExperiment < Verdict::Experiment
+  def subject_qualifies?(subject, context = nil)
+    return false if subject.even?
+    super
+  end
+end

data/test/storage/cookie_storage_test.rb

@@ -6,7 +6,9 @@ require 'fake_app'
 class CookieStorageTest < Minitest::Test
   def setup
     @storage = Verdict::Storage::CookieStorage.new.tap do |s|
-      s.cookies = ActionDispatch::Cookies::CookieJar.new(nil)
+      request = mock()
+      request.stubs(:cookies_same_site_protection).returns(proc { :none })
+      s.cookies = ActionDispatch::Cookies::CookieJar.new(request)
     end
     @experiment = Verdict::Experiment.new(:cookie_storage_test) do
       groups { group :all, 100 }

data/test/storage/redis_storage_test.rb

@@ -1,97 +1,75 @@
 require 'test_helper'
 
-class RedisStorageTest < Minitest::Test
 
-  def setup
-    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
-    @storage = storage = Verdict::Storage::RedisStorage.new(@redis)
-    @experiment = Verdict::Experiment.new(:redis_storage) do
-      qualify { |s| s == 'subject_1' }
-      groups { group :all, 100 }
-      storage storage, store_unqualified: true
-    end
-  end
-
-  def teardown
-    @redis.del('experiments/redis_storage')
-  end
+module SharedRedisStorageTests
+  attr_reader :redis, :experiment, :storage
 
   def test_store_and_retrieve_qualified_assignment
-    refute @redis.hexists(experiment_key, 'assignment_subject_1')
+    refute redis.hexists(experiment_key, 'assignment_subject_1')
 
-    new_assignment = @experiment.assign('subject_1')
+    new_assignment = experiment.assign('subject_1')
     assert new_assignment.qualified?
     refute new_assignment.returning?
 
-    assert @redis.hexists(experiment_key, 'assignment_subject_1')
+    assert redis.hexists(experiment_key, 'assignment_subject_1')
 
-    returning_assignment = @experiment.assign('subject_1')
+    returning_assignment = experiment.assign('subject_1')
     assert returning_assignment.returning?
     assert_equal new_assignment.experiment, returning_assignment.experiment
     assert_equal new_assignment.group, returning_assignment.group
   end
 
   def test_store_and_retrieve_unqualified_assignment
-    refute @redis.hexists(experiment_key, 'assignment_subject_2')
+    refute redis.hexists(experiment_key, 'assignment_subject_2')
 
-    new_assignment = @experiment.assign('subject_2')
+    new_assignment = experiment.assign('subject_2')
 
     refute new_assignment.returning?
     refute new_assignment.qualified?
-    assert @redis.hexists(experiment_key, 'assignment_subject_2')
+    assert redis.hexists(experiment_key, 'assignment_subject_2')
 
-    returning_assignment = @experiment.assign('subject_2')
+    returning_assignment = experiment.assign('subject_2')
     assert returning_assignment.returning?
     assert_equal new_assignment.experiment, returning_assignment.experiment
     assert_nil new_assignment.group
     assert_nil returning_assignment.group
   end
 
-  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_reads
-    @redis.stubs(:hget).raises(::Redis::BaseError, "Redis is down")
-    assert !@experiment.assign('subject_1').qualified?
-  end
-
-  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
-    @redis.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
-    assert !@experiment.assign('subject_1').qualified?
-  end
-
   def test_remove_subject_assignment
-    @experiment.assign('subject_3')
-    assert @redis.hexists(experiment_key, 'assignment_subject_3')
-    @experiment.remove_subject_assignment('subject_3')
-    refute @redis.hexists(experiment_key, 'assignment_subject_3')
+    experiment.assign('subject_3')
+    assert redis.hexists(experiment_key, 'assignment_subject_3')
+    experiment.remove_subject_assignment('subject_3')
+    refute redis.hexists(experiment_key, 'assignment_subject_3')
   end
 
   def test_started_at
-    refute @redis.hexists(experiment_key, "started_at")
-    a = @experiment.send(:ensure_experiment_has_started)
-    assert @redis.hexists(experiment_key, "started_at")
-    assert_equal a, @experiment.started_at
+    refute redis.hexists(experiment_key, "started_at")
+    a = experiment.send(:ensure_experiment_has_started)
+    assert redis.hexists(experiment_key, "started_at")
+    assert_equal a, experiment.started_at
   end
 
   def test_cleanup_with_scope_argument
     1000.times do |n|
-      @experiment.assign("something_#{n}")
+      experiment.assign("something_#{n}")
     end
 
-    assert_operator @redis, :exists, experiment_key
+    assert_operator redis, :exists, experiment_key
 
     Verdict.default_logger.expects(:warn).with(regexp_matches(/deprecated/))
-    @storage.cleanup(:redis_storage)
-    refute_operator @redis, :exists, experiment_key
+    storage.cleanup(:redis_storage)
+    refute_operator redis, :exists, experiment_key
   end
 
   def test_cleanup
     1000.times do |n|
-      @experiment.assign("something_#{n}")
+      experiment.assign("something_#{n}")
     end
 
-    assert_operator @redis, :exists, experiment_key
+    assert_operator redis, :exists, experiment_key
 
-    @storage.cleanup(@experiment)
-    refute_operator @redis, :exists, experiment_key
+    storage.cleanup(experiment)
+    refute_operator redis, :exists, experiment_key
   end
 
   private
@@ -100,3 +78,67 @@ class RedisStorageTest < Minitest::Test
     "experiments/redis_storage"
   end
 end
+
+class RedisStorageTest < Minitest::Test
+  include SharedRedisStorageTests
+
+  def setup
+    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    @storage = storage = Verdict::Storage::RedisStorage.new(@redis)
+    @experiment = Verdict::Experiment.new(:redis_storage) do
+      qualify { |s| s == 'subject_1' }
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+  end
+
+  def teardown
+    @redis.del('experiments/redis_storage')
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_reads
+    redis.stubs(:hget).raises(::Redis::BaseError, "Redis is down")
+    assert !experiment.assign('subject_1').qualified?
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
+    redis.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
+    assert !experiment.assign('subject_1').qualified?
+  end
+end
+
+class RedisStorageConnectionPoolTest < Minitest::Test
+  include SharedRedisStorageTests
+
+  def setup
+    @redis = ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    @redis_pool = ::ConnectionPool.new(size: 3) do
+      ::Redis.new(host: REDIS_HOST, port: REDIS_PORT)
+    end
+    @storage = storage = Verdict::Storage::RedisStorage.new(@redis_pool)
+    @experiment = Verdict::Experiment.new(:redis_storage) do
+      qualify { |s| s == 'subject_1' }
+      groups { group :all, 100 }
+      storage storage, store_unqualified: true
+    end
+  end
+
+  def teardown
+    @redis.del('experiments/redis_storage')
+  end
+
+  def test_assign_should_return_unqualified_when_redis_is_unavailable_for_writes
+    redis_mock = stub
+    redis_mock.stubs(:hget).returns(nil)
+    redis_mock.stubs(:hset).raises(::Redis::BaseError, "Redis is down")
+    @redis_pool.stubs(:with).yields(redis_mock)
+    assert !@experiment.assign('subject_1').qualified?
+  end
+
+  def test_remove_subject_assignment
+    @experiment.assign('subject_3')
+    assert @redis.hexists(experiment_key, 'assignment_subject_3')
+    @experiment.remove_subject_assignment('subject_3')
+    refute @redis.hexists(experiment_key, 'assignment_subject_3')
+  end
+end

data/test/test_helper.rb

@@ -13,6 +13,7 @@ require "mocha/setup"
 require "timecop"
 require "verdict"
 require "redis"
+require "connection_pool"
 
 REDIS_HOST = ENV['REDIS_HOST'].nil? ? '127.0.0.1' : ENV['REDIS_HOST']
 REDIS_PORT = (ENV['REDIS_PORT'].nil? ? '6379' : ENV['REDIS_PORT']).to_i

data/verdict.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("timecop")
   gem.add_development_dependency("redis")
   gem.add_development_dependency("rails")
+  gem.add_development_dependency("connection_pool")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: verdict
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-29 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Shopify Experiments classes
 email:
 - kevin.mcphillips@shopify.com
@@ -103,14 +117,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/probots.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- dev.yml
 - docs/concepts.md
 - lib/verdict.rb
 - lib/verdict/assignment.rb

data/.travis.yml

@@ -1,16 +0,0 @@
-language: ruby
-cache: bundler
-before_install: gem update bundler
-script: bundle exec rake
-rvm:
-  - 2.0
-  - 2.1
-  - 2.3.3
-  - ruby-head
-  - jruby
-matrix:
-  allow_failures:
-    - rvm: ruby-head
-    - rvm: jruby
-services:
-  - redis-server