scientist 1.4.0 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49ab4f74d43836220fbadd4692445a3cc61a13387cf87f533f39db9c58ba8544
-  data.tar.gz: d1648b2e74411130e36ddcac2f9e9bece7c7db3d65ca37deb1d7d6cf7a28096e
+  metadata.gz: 1e623b9acbede185a6ee8ca0ae2227a52707499c28bdf04e4fe8a2afcbd0f051
+  data.tar.gz: 560b1d3353829f6be217229546297fcd4102e6c47dc87fc12ffa963c5e2fa3f7
 SHA512:
-  metadata.gz: 67f99e2906499d736c4ab002273d0bf9dbea07a3413b387c12a239afbdc65a542d658f7f1d087cdf482a261698e02c439ecb673a73a6497ca2ce4e07940e1a25
-  data.tar.gz: 60479759bc916ae2b0d1c1b9bf575de872fbb5adfda098ee6fea4802384ceeed30caee4973583e751de9c5b3dce686acc1602aa7f6fade034d22b4f3ae5e7180
+  metadata.gz: 593c2807ff16f1381a0189e4babdad3ab366ad66057e4af1b083611fb204d55ae9434bf989550f8371de7586be55fe4f8a760f84f2424323a9189803583e9243
+  data.tar.gz: 1239bb86dfad9fcee2b0dc4dc2fc8458306e877afd2aa1e722b9e4db7072b71658469bf48b425959c52111fac2cc8bb5dbbb11435263a199f036ad2ee4697228

data/.travis.yml

@@ -3,10 +3,10 @@ language: ruby
 cache: bundler
 script: script/test
 rvm:
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - 2.6.1
+  - 2.6.7
+  - 2.7.3
+  - 3.0.1
+  - truffleruby-head
 before_install: gem install bundler
 addons:
  apt:

data/CONTRIBUTING.md

@@ -10,9 +10,9 @@ Contributions to this project are [released](https://help.github.com/articles/gi
 ## Submitting a pull request
 
 0. [Fork][fork] and clone the repository
-0. Create a new branch: `git checkout -b my-branch-name`
-0. Make your change, push to your fork and [submit a pull request][pr]
-0. Pat your self on the back and wait for your pull request to be reviewed and merged.
+1. Create a new branch: `git checkout -b my-branch-name`
+2. Make your change, push to your fork and [submit a pull request][pr]
+3. Pat your self on the back and wait for your pull request to be reviewed and merged.
 
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 

data/README.md

@@ -24,9 +24,9 @@ Wrap a `use` block around the code's original behavior, and wrap `try` around th
 
 * It decides whether or not to run the `try` block,
 * Randomizes the order in which `use` and `try` blocks are run,
-* Measures the durations of all behaviors,
+* Measures the durations of all behaviors in seconds,
 * Compares the result of `try` to the result of `use`,
-* Swallows (but records) any exceptions raised in the `try` block, and
+* Swallow and record exceptions raised in the `try` block when overriding `raised`, and
 * Publishes all this information.
 
 The `use` block is called the **control**. The `try` block is called the **candidate**.
@@ -62,7 +62,7 @@ class MyExperiment
 
   attr_accessor :name
 
-  def initialize(name:)
+  def initialize(name)
     @name = name
   end
 
@@ -71,20 +71,21 @@ class MyExperiment
     true
   end
 
+  def raised(operation, error)
+    # see "In a Scientist callback" below
+    p "Operation '#{operation}' failed with error '#{error.inspect}'"
+    super # will re-raise
+  end
+
   def publish(result)
     # see "Publishing results" below
     p result
   end
 end
-
-# replace `Scientist::Default` as the default implementation
-module Scientist::Experiment
-  def self.new(name)
-    MyExperiment.new(name: name)
-  end
-end
 ```
 
+When `Scientist::Experiment` is included in a class, it automatically sets it as the default implementation via `Scientist::Experiment.set_default`. This `set_default` call is skipped if you include `Scientist::Experiment` in a module.
+
 Now calls to the `science` helper will load instances of `MyExperiment`.
 
 ### Controlling comparison
@@ -108,6 +109,38 @@ class MyWidget
 end
 ```
 
+If either the control block or candidate block raises an error, Scientist compares the two observations' classes and messages using `==`. To override this behavior, use `compare_error` to define how to compare observed errors instead:
+
+```ruby
+class MyWidget
+  include Scientist
+
+  def slug_from_login(login)
+    science "slug_from_login" do |e|
+      e.use { User.slug_from_login login }         # returns String instance or ArgumentError
+      e.try { UserService.slug_from_login login }  # returns String instance or ArgumentError
+
+      compare_error_message_and_class = -> (control, candidate) do
+        control.class == candidate.class && 
+        control.message == candidate.message
+      end
+
+      compare_argument_errors = -> (control, candidate) do
+        control.class == ArgumentError &&
+        candidate.class == ArgumentError &&
+        control.message.start_with?("Input has invalid characters") &&
+        candidate.message.start_with?("Invalid characters in input") 
+      end
+
+      e.compare_error do |control, candidate|
+        compare_error_message_and_class.call(control, candidate) ||
+        compare_argument_errors.call(control, candidate)
+      end
+    end
+  end
+end
+```
+
 ### Adding context
 
 Results aren't very useful without some way to identify them. Use the `context` method to add to or retrieve the context for an experiment:
@@ -227,7 +260,7 @@ def admin?(user)
 end
 ```
 
-The ignore blocks are only called if the *values* don't match. If one observation raises an exception and the other doesn't, it's always considered a mismatch. If both observations raise different exceptions, that is also considered a mismatch.
+The ignore blocks are only called if the *values* don't match. Unless a `compare_error` comparator is defined, two cases are considered mismatches: a) one observation raising an exception and the other not, b) observations raising exceptions with different classes or messages.
 
 ### Enabling/disabling experiments
 
@@ -256,7 +289,7 @@ class MyExperiment
 
   attr_accessor :name, :percent_enabled
 
-  def initialize(name:)
+  def initialize(name)
     @name = name
     @percent_enabled = 100
   end
@@ -394,6 +427,8 @@ Scientist rescues and tracks _all_ exceptions raised in a `try` or `use` block,
 Scientist::Observation::RESCUES.replace [StandardError]
 ```
 
+**Timeout ⏲️**: If you're introducing a candidate that could possibly timeout, use caution. ⚠️ While Scientist rescues all exceptions that occur in the candidate block, it *does not* protect you from timeouts, as doing so would be complicated. It would likely require running the candidate code in a background job and tracking the time of a request. We feel the cost of this complexity would outweigh the benefit, so make sure that your code doesn't cause timeouts. This risk can be reduced by running the experiment on a low percentage so that users can (most likely) bypass the experiment by refreshing the page if they hit a timeout. See [Ramping up experiments](#ramping-up-experiments) below for how details on how to set the percentage for your experiment.
+
 #### In a Scientist callback
 
 If an exception is raised within any of Scientist's internal helpers, like `publish`, `compare`, or `clean`, the `raised` method is called with the symbol name of the internal operation that failed and the exception that was raised. The default behavior of `Scientist::Default` is to simply re-raise the exception. Since this halts the experiment entirely, it's often a better idea to handle this error and continue so the experiment as a whole isn't canceled entirely:
@@ -537,6 +572,8 @@ Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs t
 - [tomiaijo/scientist](https://github.com/tomiaijo/scientist) (C++)
 - [trello/scientist](https://github.com/trello/scientist) (node.js)
 - [ziyasal/scientist.js](https://github.com/ziyasal/scientist.js) (node.js, ES6)
+- [TrueWill/tzientist](https://github.com/TrueWill/tzientist) (node.js, TypeScript)
+- [TrueWill/paleontologist](https://github.com/TrueWill/paleontologist) (Deno, TypeScript)
 - [yeller/laboratory](https://github.com/yeller/laboratory) (Clojure)
 - [lancew/Scientist](https://github.com/lancew/Scientist) (Perl 5)
 - [lancew/ScientistP6](https://github.com/lancew/ScientistP6) (Perl 6)
@@ -546,6 +583,9 @@ Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs t
 - [jelmersnoeck/experiment](https://github.com/jelmersnoeck/experiment) (Go)
 - [spoptchev/scientist](https://github.com/spoptchev/scientist) (Kotlin / Java)
 - [junkpiano/scientist](https://github.com/junkpiano/scientist) (Swift)
+- [serverless scientist](http://serverlessscientist.com/) (AWS Lambda)
+- [fightmegg/scientist](https://github.com/fightmegg/scientist) (TypeScript, Browser / Node.js)
+- [MisterSpex/misterspex-scientist](https://github.com/MisterSpex/misterspex-scientist) (Java, no dependencies)
 
 ## Maintainers
 

data/lib/scientist/experiment.rb

@@ -9,12 +9,22 @@ module Scientist::Experiment
   # If this is nil, raise_on_mismatches class attribute is used instead.
   attr_accessor :raise_on_mismatches
 
-  # Create a new instance of a class that implements the Scientist::Experiment
-  # interface.
-  #
-  # Override this method directly to change the default implementation.
+  def self.included(base)
+    self.set_default(base) if base.instance_of?(Class)
+    base.extend RaiseOnMismatch
+  end
+
+  # Instantiate a new experiment (using the class given to the .set_default method).
   def self.new(name)
-    Scientist::Default.new(name)
+    (@experiment_klass || Scientist::Default).new(name)
+  end
+
+  # Configure Scientist to use the given class for all future experiments
+  # (must implement the Scientist::Experiment interface).
+  #
+  # Called automatically when new experiments are defined.
+  def self.set_default(klass)
+    @experiment_klass = klass
   end
 
   # A mismatch, raised when raise_on_mismatches is enabled.
@@ -67,10 +77,6 @@ module Scientist::Experiment
     end
   end
 
-  def self.included(base)
-    base.extend RaiseOnMismatch
-  end
-
   # Define a block of code to run before an experiment begins, if the experiment
   # is enabled.
   #
@@ -78,6 +84,7 @@ module Scientist::Experiment
   #
   # Returns the configured block.
   def before_run(&block)
+    marshalize(block)
     @_scientist_before_run = block
   end
 
@@ -93,6 +100,7 @@ module Scientist::Experiment
   #
   # Returns the configured block.
   def clean(&block)
+    marshalize(block)
     @_scientist_cleaner = block
   end
 
@@ -125,9 +133,21 @@ module Scientist::Experiment
   #
   # Returns the block.
   def compare(*args, &block)
+    marshalize(block)
     @_scientist_comparator = block
   end
 
+  # A block which compares two experimental errors.
+  #
+  # The block must take two arguments, the control Error and a candidate Error,
+  # and return true or false.
+  #
+  # Returns the block.
+  def compare_errors(*args, &block)
+    marshalize(block)
+    @_scientist_error_comparator = block
+  end
+
   # A Symbol-keyed Hash of extra experiment data.
   def context(context = nil)
     @_scientist_context ||= {}
@@ -143,6 +163,7 @@ module Scientist::Experiment
   #
   # This can be called more than once with different blocks to use.
   def ignore(&block)
+    marshalize(block)
     @_scientist_ignores ||= []
     @_scientist_ignores << block
   end
@@ -171,13 +192,9 @@ module Scientist::Experiment
     "experiment"
   end
 
-  # Internal: compare two observations, using the configured compare block if present.
+  # Internal: compare two observations, using the configured compare and compare_errors lambdas if present.
   def observations_are_equivalent?(a, b)
-    if @_scientist_comparator
-      a.equivalent_to?(b, &@_scientist_comparator)
-    else
-      a.equivalent_to? b
-    end
+    a.equivalent_to? b, @_scientist_comparator, @_scientist_error_comparator
   rescue StandardError => ex
     raised :compare, ex
     false
@@ -216,17 +233,7 @@ module Scientist::Experiment
       @_scientist_before_run.call
     end
 
-    observations = []
-
-    behaviors.keys.shuffle.each do |key|
-      block = behaviors[key]
-      fabricated_duration = @_scientist_fabricated_durations && @_scientist_fabricated_durations[key]
-      observations << Scientist::Observation.new(key, self, fabricated_duration: fabricated_duration, &block)
-    end
-
-    control = observations.detect { |o| o.name == name }
-
-    result = Scientist::Result.new self, observations, control
+    result = generate_result(name)
 
     begin
       publish(result)
@@ -242,15 +249,14 @@ module Scientist::Experiment
       end
     end
 
-    if control.raised?
-      raise control.exception
-    else
-      control.value
-    end
+    control = result.control
+    raise control.exception if control.raised?
+    control.value
   end
 
   # Define a block that determines whether or not the experiment should run.
   def run_if(&block)
+    marshalize(block)
     @_scientist_run_if_block = block
   end
 
@@ -276,6 +282,7 @@ module Scientist::Experiment
 
   # Register a named behavior for this experiment, default "candidate".
   def try(name = nil, &block)
+    marshalize(block)
     name = (name || "candidate").to_s
 
     if behaviors.include?(name)
@@ -287,6 +294,7 @@ module Scientist::Experiment
 
   # Register the control behavior for this experiment.
   def use(&block)
+    marshalize(block)
     try "control", &block
   end
 
@@ -304,4 +312,33 @@ module Scientist::Experiment
   def fabricate_durations_for_testing_purposes(fabricated_durations = {})
     @_scientist_fabricated_durations = fabricated_durations
   end
+
+  # Internal: Generate the observations and create the result from those and the control.
+  def generate_result(name)
+    observations = []
+
+    behaviors.keys.shuffle.each do |key|
+      block = behaviors[key]
+      fabricated_duration = @_scientist_fabricated_durations && @_scientist_fabricated_durations[key]
+      observations << Scientist::Observation.new(key, self, fabricated_duration: fabricated_duration, &block)
+    end
+
+    control = observations.detect { |o| o.name == name }
+    Scientist::Result.new(self, observations, control)
+  end
+
+  private
+
+  # In order to support marshaling, we have to make the procs marshalable. Some
+  # CI providers attempt to marshal Scientist mismatch errors so that they can
+  # be sent out to different places (logs, etc.) The mismatch errors contain
+  # code from the experiment. This code contains Procs - which can't be
+  # marshaled until we run the following code.
+  def marshalize(block)
+    unless block.respond_to?(:_dump) || block.respond_to?(:_dump_data)
+      def block._dump(_)
+        to_s
+      end
+    end
+  end
 end

data/lib/scientist/observation.rb

@@ -8,9 +8,6 @@ class Scientist::Observation
   # The experiment this observation is for
   attr_reader :experiment
 
-  # The instant observation began.
-  attr_reader :now
-
   # The String name of the behavior.
   attr_reader :name
 
@@ -26,7 +23,6 @@ class Scientist::Observation
   def initialize(name, experiment, fabricated_duration: nil, &block)
     @name       = name
     @experiment = experiment
-    @now        = Time.now
 
     starting = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) unless fabricated_duration
     begin
@@ -49,25 +45,34 @@ class Scientist::Observation
 
   # Is this observation equivalent to another?
   #
-  # other      - the other Observation in question
-  # comparator - an optional comparison block. This observation's value and the
-  #              other observation's value are yielded to this to determine
-  #              their equivalency. Block should return true/false.
+  # other            - the other Observation in question
+  # comparator       - an optional comparison proc. This observation's value and the
+  #                    other observation's value are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
+  # error_comparator - an optional comparison proc. This observation's Error and the
+  #                    other observation's Error are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
   #
   # Returns true if:
   #
   # * The values of the observation are equal (using `==`)
   # * The values of the observations are equal according to a comparison
-  #   block, if given
+  #   proc, if given
+  # * The exceptions raised by the obeservations are equal according to the
+  #   error comparison proc, if given.
   # * Both observations raised an exception with the same class and message.
   #
   # Returns false otherwise.
-  def equivalent_to?(other, &comparator)
+  def equivalent_to?(other, comparator=nil, error_comparator=nil)
     return false unless other.is_a?(Scientist::Observation)
 
     if raised? || other.raised?
-      return other.exception.class == exception.class &&
-        other.exception.message == exception.message
+      if error_comparator
+        return error_comparator.call(exception, other.exception)
+      else
+        return other.exception.class == exception.class &&
+          other.exception.message == exception.message
+      end
     end
 
     if comparator

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "1.4.0"
+  VERSION = "1.6.2"
 end

data/script/release

@@ -12,10 +12,10 @@ cd $(dirname "$0")/..
 rm -rf scientist-*.gem
 gem build -q scientist.gemspec
 
-# Make sure we're on the master branch.
+# Make sure we're on the main branch.
 
-(git branch --no-color | grep -q '* master') || {
-  echo "Only release from the master branch."
+(git branch --no-color | grep -q '* main') || {
+  echo "Only release from the main branch."
   exit 1
 }
 
@@ -35,4 +35,4 @@ git fetch -t origin
 # Tag it and bag it.
 
 gem push scientist-*.gem && git tag "$tag" &&
-  git push origin master && git push origin "$tag"
+  git push origin main && git push origin "$tag"

data/test/scientist/experiment_test.rb

@@ -2,6 +2,9 @@ describe Scientist::Experiment do
   class Fake
     include Scientist::Experiment
 
+    # Undo auto-config magic / preserve default behavior of Scientist::Experiment.new
+    Scientist::Experiment.set_default(nil)
+
     def initialize(*args)
     end
 
@@ -28,6 +31,24 @@ describe Scientist::Experiment do
     @ex = Fake.new
   end
 
+  it "sets the default on inclusion" do
+    klass = Class.new do
+      include Scientist::Experiment
+
+      def initialize(name)
+      end
+    end
+
+    assert_kind_of klass, Scientist::Experiment.new("hello")
+
+    Scientist::Experiment.set_default(nil)
+  end
+
+  it "doesn't set the default on inclusion when it's a module" do
+    Module.new { include Scientist::Experiment }
+    assert_kind_of Scientist::Default, Scientist::Experiment.new("hello")
+  end
+
   it "has a default implementation" do
     ex = Scientist::Experiment.new("hello")
     assert_kind_of Scientist::Default, ex
@@ -180,6 +201,18 @@ describe Scientist::Experiment do
     assert @ex.published_result.matched?
   end
 
+  it "compares errors with an error comparator block if provided" do
+    @ex.compare_errors { |a, b| a.class == b.class }
+    @ex.use { raise "foo" }
+    @ex.try { raise "bar" }
+
+    resulting_error = assert_raises RuntimeError do
+      @ex.run
+    end
+    assert_equal "foo", resulting_error.message
+    assert @ex.published_result.matched?
+  end
+
   it "knows how to compare two experiments" do
     a = Scientist::Observation.new(@ex, "a") { 1 }
     b = Scientist::Observation.new(@ex, "b") { 2 }
@@ -458,6 +491,27 @@ describe Scientist::Experiment do
       assert_raises(Scientist::Experiment::MismatchError) { runner.call }
     end
 
+    it "can be marshaled" do
+      Fake.raise_on_mismatches = true
+      @ex.before_run { "some block" }
+      @ex.clean { "some block" }
+      @ex.compare_errors { "some block" }
+      @ex.ignore { false }
+      @ex.run_if { "some block" }
+      @ex.try { "candidate" }
+      @ex.use { "control" }
+      @ex.compare { |control, candidate| control == candidate }
+
+      mismatch = nil
+      begin
+        @ex.run
+      rescue Scientist::Experiment::MismatchError => e
+        mismatch = e
+      end
+
+      assert_kind_of(String, Marshal.dump(mismatch))
+    end
+
     describe "#raise_on_mismatches?" do
       it "raises when there is a mismatch if the experiment instance's raise on mismatches is enabled" do
         Fake.raise_on_mismatches = false
@@ -546,7 +600,7 @@ candidate:
         assert_equal "  \"value\"", lines[2]
         assert_equal "candidate:", lines[3]
         assert_equal "  #<RuntimeError: error>", lines[4]
-        assert_match %r(    test/scientist/experiment_test.rb:\d+:in `block), lines[5]
+        assert_match %r(test/scientist/experiment_test.rb:\d+:in `block), lines[5]
       end
     end
   end

data/test/scientist/observation_test.rb

@@ -80,7 +80,7 @@ describe Scientist::Observation do
     refute x.equivalent_to?(y)
   end
 
-  FirstErrror = Class.new(StandardError)
+  FirstError = Class.new(StandardError)
   SecondError = Class.new(StandardError)
 
   it "compares exception classes" do
@@ -92,22 +92,46 @@ describe Scientist::Observation do
     refute x.equivalent_to?(y)
   end
 
-  it "compares values using a comparator block" do
+  it "compares values using a comparator proc" do
     a = Scientist::Observation.new("test", @experiment) { 1 }
     b = Scientist::Observation.new("test", @experiment) { "1" }
 
     refute a.equivalent_to?(b)
-    assert a.equivalent_to?(b) { |x, y| x.to_s == y.to_s }
+
+    compare_on_string = -> (x, y) { x.to_s == y.to_s }
+
+    assert a.equivalent_to?(b, compare_on_string)
 
     yielded = []
-    a.equivalent_to?(b) do |x, y|
+    compare_appends = -> (x, y) do
       yielded << x
       yielded << y
       true
     end
+    a.equivalent_to?(b, compare_appends)
+
     assert_equal [a.value, b.value], yielded
   end
 
+  it "compares exceptions using an error comparator proc" do
+    x = Scientist::Observation.new("test", @experiment) { raise FirstError, "error" }
+    y = Scientist::Observation.new("test", @experiment) { raise SecondError, "error" }
+    z = Scientist::Observation.new("test", @experiment) { raise FirstError, "ERROR" }
+
+    refute x.equivalent_to?(z)
+    refute x.equivalent_to?(y)
+
+    compare_on_class = -> (error, other_error) {
+      error.class == other_error.class
+    }
+    compare_on_message = -> (error, other_error) {
+      error.message == other_error.message
+    }
+
+    assert x.equivalent_to?(z, nil, compare_on_class)
+    assert x.equivalent_to?(y, nil, compare_on_message)
+  end
+
   describe "#cleaned_value" do
     it "returns the observation's value by default" do
       a = Scientist::Observation.new("test", @experiment) { 1 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scientist
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.2
 platform: ruby
 authors:
 - GitHub Open Source
@@ -9,10 +9,10 @@ authors:
 - Rick Bradley
 - Jesse Toth
 - Nathan Witmer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-20 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -80,7 +80,7 @@ homepage: https://github.com/github/scientist
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -95,8 +95,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Carefully test, measure, and track refactored code.
 test_files: