verdict 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec6df6bdd38dc947ff1072e1dce844ab4d3ddd522701c8de578a35b4c9c51be8
-  data.tar.gz: e6b62b2c579a363f37526079389bea4cb16eb28ed7573dec2d3fa8f8c74ee4ed
+  metadata.gz: f05448f375933cba0d01c10a4e302b684242297e779d96cd3bf45cff2f6fea74
+  data.tar.gz: d8782ab103377d758a550f36f16fa40cec2caa2662333a99d180aab637985126
 SHA512:
-  metadata.gz: 1399fa37c4575888ffb83aea54469784db11ceef1ebc99f5c1fa5de75b936f460b1f34dc717c28f4f19a6e969c3a85a3faad022119113cd94f754ad61b4fdb85
-  data.tar.gz: 7f1836b8d23ca79a4338f0a4f4ed244ebe7bdbd8b9f03529f2cb00e110c280b1033e112928da900d805112eb0036e0f74278ea021514016541accf01d8c3308c
+  metadata.gz: 14317430460d1fcb88748507d20fd8be9f0a271d434951c6deb4b89203eeb3a5675a7c88e5e1d79bd6404c275bc3a9a3b35558a09df51284c100613963c4ba51
+  data.tar.gz: 39ad70804de5c962785117525ccd668155efd228860735de13569316d22e6e3ed65e050842753c7283bc21c299ee237a00885686d6963a94e3130594f5c61b21

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## v0.14.0
+* Add optional experiment definition method `schedule_stop_new_assignment_timestamp` to support limiting experiment's assignment lifetime with another pre-determined time interval. It allows users to have an assignment cooldown period for stable analysis of the experiment results. Experiment's lifetime now becomes: start experiment -> stop new assignments -> end experiment. 
+
 ## v0.13.0
 
 * Add optional experiment definition methods `schedule_start_timestamp` and `schedule_end_timestamp` to support limiting experiment's lifetime in a pre-determined time interval.

data/lib/verdict/experiment.rb

@@ -52,9 +52,17 @@ class Verdict::Experiment
     return self
   end
 
-  # Optional: Together with the "end timestamp", limits the experiment run timeline within
-  # the given time interval. When experiment is not scheduled, subject switch returns nil.
-  # This is useful when the experimenter requires the experiment to run in a strict timeline.
+  # Optional: Together with the "end_timestamp" and "stop_new_assignment_timestamp", limits the experiment run timeline within
+  # the given time interval.
+  #
+  # Timestamps definitions:
+  # start_timestamp: Experiment's start time. No assignments are made i.e. switch will return nil before this timestamp.
+  # stop_new_assignment_timestamp: Experiment's new assignment stop time. No new assignments are made
+  # i.e. switch returns nil for new assignments but the existing assignments are preserved.
+  # end_timestamp: Experiment's end time. No assignments are made i.e. switch returns nil after this timestamp.
+  #
+  # Experiment run timeline:
+  # start_timestamp -> (new assignments occur) -> stop_new_assignment_timestamp -> (no new assignments occur) -> end_timestamp
   def schedule_start_timestamp(timestamp)
     @schedule_start_timestamp = timestamp
   end
@@ -63,6 +71,10 @@ class Verdict::Experiment
     @schedule_end_timestamp = timestamp
   end
 
+  def schedule_stop_new_assignment_timestamp(timestamp)
+    @schedule_stop_new_assignment_timestamp = timestamp
+  end
+
   def rollout_percentage(percentage, rollout_group_name = :enabled)
     groups(Verdict::Segmenters::RolloutSegmenter) do
       group rollout_group_name, percentage
@@ -136,7 +148,7 @@ class Verdict::Experiment
     subject_identifier = retrieve_subject_identifier(subject)
     assignment = if previous_assignment
                    previous_assignment
-                 elsif subject_qualifies?(subject, context)
+                 elsif subject_qualifies?(subject, context) && is_make_new_assignments?
                    group = segmenter.assign(subject_identifier, subject, context)
                    subject_assignment(subject, group, nil, group.nil?)
                  else
@@ -268,12 +280,16 @@ class Verdict::Experiment
   private
 
   def is_scheduled?
-    if @schedule_start_timestamp and @schedule_start_timestamp > Time.now
+    if @schedule_start_timestamp && @schedule_start_timestamp > Time.now
       return false
     end
-    if @schedule_end_timestamp and @schedule_end_timestamp < Time.now
+    if @schedule_end_timestamp && @schedule_end_timestamp <= Time.now
       return false
     end
     return true
   end
+
+  def is_make_new_assignments?
+    return !(@schedule_stop_new_assignment_timestamp && @schedule_stop_new_assignment_timestamp <= Time.now)
+  end
 end

data/lib/verdict/version.rb

@@ -1,3 +1,3 @@
 module Verdict
-  VERSION = "0.13.0"
+  VERSION = "0.14.0"
 end

data/test/experiment_test.rb

@@ -565,6 +565,82 @@ class ExperimentTest < Minitest::Test
     end
   end
 
+  def test_is_stop_new_assignments
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+      schedule_stop_new_assignment_timestamp Time.new(2020, 1, 15)
+    end
+
+    # new assignments stopped after the stop timestamp
+    Timecop.freeze(Time.new(2020, 1, 16)) do
+      assert !e.send(:is_make_new_assignments?)
+      assert_nil e.switch(1)
+    end
+    # new assignments didn't stop before the stop timestamp
+    Timecop.freeze(Time.new(2020, 1, 3)) do
+      assert e.send(:is_make_new_assignments?)
+      assert :a, e.switch(2)
+    end
+  end
+
+  def test_switch_preserves_old_assignments_after_stop_new_assignments_timestamp
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+    end
+
+    assert_equal :a, e.switch(1)
+
+    e.schedule_stop_new_assignment_timestamp Time.new(2020, 4, 15)
+
+    # switch respects to stop new assignment timestamp, old assignment preserves, new assignment returns nil
+    Timecop.freeze(Time.new(2020, 4, 16)) do
+      assert !e.send(:is_make_new_assignments?)
+      # old assignment stay the same
+      assert_equal :a, e.switch(1)
+      # new assignment returns nil
+      assert_nil e.switch(2)
+    end
+  end
+
+  def test_schedule_start_timestamp_and_stop_new_assignemnt_timestamp_are_inclusive_but_end_timestamp_is_exclusive
+    e = Verdict::Experiment.new('test') do
+      groups do
+        group :a, :half
+        group :b, :half
+      end
+
+      schedule_start_timestamp Time.new(2020, 1, 1)
+      schedule_stop_new_assignment_timestamp Time.new(2020, 1, 15)
+      schedule_end_timestamp Time.new(2020, 1, 31)
+    end
+
+    # start_timestamp is included
+    Timecop.freeze(Time.new(2020, 1, 1)) do
+      assert e.send(:is_scheduled?)
+      assert_equal :a, e.switch(1)
+    end
+
+    # stop_new_assignment_timestamp is included
+    Timecop.freeze(Time.new(2020, 1, 15)) do
+      assert !e.send(:is_make_new_assignments?)
+      # old assignment preserved
+      assert_equal :a, e.switch(1)
+      # new assignment returns nil
+      assert_nil e.switch(2)
+    end
+
+    # end_timestamp is excluded
+    Timecop.freeze(Time.new(2020, 1, 31)) do
+      assert !e.send(:is_scheduled?)
+      assert_nil e.switch(1)
+    end
+  end
   private
 
   def redis

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: verdict
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-03 00:00:00.000000000 Z
+date: 2020-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest