scientist 1.6.0 → 1.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f03841ae9bccb8d7b979971d5f6a38482f57bbc81af82d0a710479ccc3f16fbb
-  data.tar.gz: c0f81d696918e302462baf1bd918cd25179a99045d48082fb162e1fdb2fa2280
+  metadata.gz: 5553615794cee9e0fbdb4c22b3dd528885b30a75c9c8c2bd1d5d4f8ebd4bc0ba
+  data.tar.gz: ff1c3e2b2f23defc19ad663df792da483926406aa98261d403fa7c008c03be85
 SHA512:
-  metadata.gz: f1680e6564a6e83b3dde55f6757f3e7c953a70b3d748aa0e7ca2a54b5a33a4b3a947a8f20aa52790a41e6a66e36627e4e851f627625c0aaf9fd4cf41b94ace8c
-  data.tar.gz: 75ba381053b73e9418ed2ab5608c70c92a55990c938a1c25f4324169ff560a8feea6827ef909e50b9fc3baf5a5eb49f3b7b26ed4e3f9799007b80a71c7543c86
+  metadata.gz: ffef7682cdd59612ca44b33ba3678467251c2ccabfdecda41022cc95c0d07673abd38ca3a33c357fa40358398691ba0f8ecf8ec04ddff14800a25a915f011f4d
+  data.tar.gz: 06e06b1428967829d9b330eba28bd1bd2ce47eb5b12b6317eb6c6e8ea53e4e6434337803b88fdadbccdcb23a1b8cafe2fbe98a38756ea53d81ed21afe0ec369e

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+on: [push, pull_request]
+permissions:
+  actions: none
+  checks: write
+  contents: read
+  deployments: none
+  issues: none
+  packages: none
+  pull-requests: none
+  repository-projects: none
+  security-events: none
+  statuses: write
+jobs:
+  build:
+    name: ruby-${{ matrix.ruby_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version:
+          - "2.6"
+          - "2.7"
+          - "3.0"
+          - "3.1"
+          - "3.2"
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby_version }}
+          bundler-cache: true
+      - run: bundle exec rake

data/CONTRIBUTING.md

@@ -10,9 +10,9 @@ Contributions to this project are [released](https://help.github.com/articles/gi
 ## Submitting a pull request
 
 0. [Fork][fork] and clone the repository
-0. Create a new branch: `git checkout -b my-branch-name`
-0. Make your change, push to your fork and [submit a pull request][pr]
-0. Pat your self on the back and wait for your pull request to be reviewed and merged.
+1. Create a new branch: `git checkout -b my-branch-name`
+2. Make your change, push to your fork and [submit a pull request][pr]
+3. Pat your self on the back and wait for your pull request to be reviewed and merged.
 
 Here are a few things you can do that will increase the likelihood of your pull request being accepted:
 

data/README.md

@@ -1,6 +1,6 @@
 # Scientist!
 
-A Ruby library for carefully refactoring critical paths. [![Build Status](https://travis-ci.org/github/scientist.svg?branch=master)](https://travis-ci.org/github/scientist) [![Coverage Status](https://coveralls.io/repos/github/github/scientist/badge.svg?branch=master)](https://coveralls.io/github/github/scientist?branch=master)
+A Ruby library for carefully refactoring critical paths. [![Build Status](https://github.com/github/scientist/actions/workflows/ci.yml/badge.svg)](https://github.com/github/scientist/actions/workflows/ci.yml)
 
 ## How do I science?
 
@@ -84,7 +84,7 @@ class MyExperiment
 end
 ```
 
-When `Scientist::Experiment` is included in a class, it automatically sets it as the default implementation via `Scientist::Experiment.set_default`. This `set_default` call is is skipped if you include `Scientist::Experiment` in a module.
+When `Scientist::Experiment` is included in a class, it automatically sets it as the default implementation via `Scientist::Experiment.set_default`. This `set_default` call is skipped if you include `Scientist::Experiment` in a module.
 
 Now calls to the `science` helper will load instances of `MyExperiment`.
 
@@ -109,6 +109,38 @@ class MyWidget
 end
 ```
 
+If either the control block or candidate block raises an error, Scientist compares the two observations' classes and messages using `==`. To override this behavior, use `compare_errors` to define how to compare observed errors instead:
+
+```ruby
+class MyWidget
+  include Scientist
+
+  def slug_from_login(login)
+    science "slug_from_login" do |e|
+      e.use { User.slug_from_login login }         # returns String instance or ArgumentError
+      e.try { UserService.slug_from_login login }  # returns String instance or ArgumentError
+
+      compare_error_message_and_class = -> (control, candidate) do
+        control.class == candidate.class && 
+        control.message == candidate.message
+      end
+
+      compare_argument_errors = -> (control, candidate) do
+        control.class == ArgumentError &&
+        candidate.class == ArgumentError &&
+        control.message.start_with?("Input has invalid characters") &&
+        candidate.message.start_with?("Invalid characters in input") 
+      end
+
+      e.compare_errors do |control, candidate|
+        compare_error_message_and_class.call(control, candidate) ||
+        compare_argument_errors.call(control, candidate)
+      end
+    end
+  end
+end
+```
+
 ### Adding context
 
 Results aren't very useful without some way to identify them. Use the `context` method to add to or retrieve the context for an experiment:
@@ -228,7 +260,7 @@ def admin?(user)
 end
 ```
 
-The ignore blocks are only called if the *values* don't match. If one observation raises an exception and the other doesn't, it's always considered a mismatch. If both observations raise different exceptions, that is also considered a mismatch.
+The ignore blocks are only called if the *values* don't match. Unless a `compare_errors` comparator is defined, two cases are considered mismatches: a) one observation raising an exception and the other not, b) observations raising exceptions with different classes or messages.
 
 ### Enabling/disabling experiments
 
@@ -395,6 +427,8 @@ Scientist rescues and tracks _all_ exceptions raised in a `try` or `use` block,
 Scientist::Observation::RESCUES.replace [StandardError]
 ```
 
+**Timeout ⏲️**: If you're introducing a candidate that could possibly timeout, use caution. ⚠️ While Scientist rescues all exceptions that occur in the candidate block, it *does not* protect you from timeouts, as doing so would be complicated. It would likely require running the candidate code in a background job and tracking the time of a request. We feel the cost of this complexity would outweigh the benefit, so make sure that your code doesn't cause timeouts. This risk can be reduced by running the experiment on a low percentage so that users can (most likely) bypass the experiment by refreshing the page if they hit a timeout. See [Ramping up experiments](#ramping-up-experiments) below for how details on how to set the percentage for your experiment.
+
 #### In a Scientist callback
 
 If an exception is raised within any of Scientist's internal helpers, like `publish`, `compare`, or `clean`, the `raised` method is called with the symbol name of the internal operation that failed and the exception that was raised. The default behavior of `Scientist::Default` is to simply re-raise the exception. Since this halts the experiment entirely, it's often a better idea to handle this error and continue so the experiment as a whole isn't canceled entirely:
@@ -551,6 +585,7 @@ Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs t
 - [junkpiano/scientist](https://github.com/junkpiano/scientist) (Swift)
 - [serverless scientist](http://serverlessscientist.com/) (AWS Lambda)
 - [fightmegg/scientist](https://github.com/fightmegg/scientist) (TypeScript, Browser / Node.js)
+- [MisterSpex/misterspex-scientist](https://github.com/MisterSpex/misterspex-scientist) (Java, no dependencies)
 
 ## Maintainers
 

data/Rakefile

@@ -0,0 +1,7 @@
+require "rake/testtask"
+
+task default: "test"
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/test_helper.rb', 'test/**/*_test.rb']
+end

data/lib/scientist/experiment.rb

@@ -87,6 +87,16 @@ module Scientist::Experiment
     @_scientist_before_run = block
   end
 
+  # Define a block of code to run after an experiment completes, if the experiment
+  # is enabled.
+  #
+  # The block takes one argument, the Scientist::Result containing experiment results.
+  #
+  # Returns the configured block.
+  def after_run(&block)
+    @_scientist_after_run = block
+  end
+
   # A Hash of behavior blocks, keyed by String name. Register behavior blocks
   # with the `try` and `use` methods.
   def behaviors
@@ -134,6 +144,16 @@ module Scientist::Experiment
     @_scientist_comparator = block
   end
 
+  # A block which compares two experimental errors.
+  #
+  # The block must take two arguments, the control Error and a candidate Error,
+  # and return true or false.
+  #
+  # Returns the block.
+  def compare_errors(*args, &block)
+    @_scientist_error_comparator = block
+  end
+
   # A Symbol-keyed Hash of extra experiment data.
   def context(context = nil)
     @_scientist_context ||= {}
@@ -177,13 +197,9 @@ module Scientist::Experiment
     "experiment"
   end
 
-  # Internal: compare two observations, using the configured compare block if present.
+  # Internal: compare two observations, using the configured compare and compare_errors lambdas if present.
   def observations_are_equivalent?(a, b)
-    if @_scientist_comparator
-      a.equivalent_to?(b, &@_scientist_comparator)
-    else
-      a.equivalent_to? b
-    end
+    a.equivalent_to? b, @_scientist_comparator, @_scientist_error_comparator
   rescue StandardError => ex
     raised :compare, ex
     false
@@ -224,6 +240,10 @@ module Scientist::Experiment
 
     result = generate_result(name)
 
+    if @_scientist_after_run
+      @_scientist_after_run.call(result)
+    end
+
     begin
       publish(result)
     rescue StandardError => ex
@@ -312,4 +332,20 @@ module Scientist::Experiment
     control = observations.detect { |o| o.name == name }
     Scientist::Result.new(self, observations, control)
   end
+
+  private
+
+  # In order to support marshaling, we have to make the procs marshalable. Some
+  # CI providers attempt to marshal Scientist mismatch errors so that they can
+  # be sent out to different places (logs, etc.) The mismatch errors contain
+  # code from the experiment. This code contains procs. These procs prevent the
+  # error from being marshaled. To fix this, we simple exclude the procs from
+  # the data that we marshal.
+  def marshal_dump
+    [@name, @result, @raise_on_mismatches]
+  end
+
+  def marshal_load
+    @name, @result, @raise_on_mismatches = array
+  end
 end

data/lib/scientist/observation.rb

@@ -45,25 +45,34 @@ class Scientist::Observation
 
   # Is this observation equivalent to another?
   #
-  # other      - the other Observation in question
-  # comparator - an optional comparison block. This observation's value and the
-  #              other observation's value are yielded to this to determine
-  #              their equivalency. Block should return true/false.
+  # other            - the other Observation in question
+  # comparator       - an optional comparison proc. This observation's value and the
+  #                    other observation's value are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
+  # error_comparator - an optional comparison proc. This observation's Error and the
+  #                    other observation's Error are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
   #
   # Returns true if:
   #
   # * The values of the observation are equal (using `==`)
   # * The values of the observations are equal according to a comparison
-  #   block, if given
+  #   proc, if given
+  # * The exceptions raised by the observations are equal according to the
+  #   error comparison proc, if given.
   # * Both observations raised an exception with the same class and message.
   #
   # Returns false otherwise.
-  def equivalent_to?(other, &comparator)
+  def equivalent_to?(other, comparator=nil, error_comparator=nil)
     return false unless other.is_a?(Scientist::Observation)
 
     if raised? || other.raised?
-      return other.exception.class == exception.class &&
-        other.exception.message == exception.message
+      if error_comparator
+        return error_comparator.call(exception, other.exception)
+      else
+        return other.exception.class == exception.class &&
+          other.exception.message == exception.message
+      end
     end
 
     if comparator

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "1.6.0"
+  VERSION = '1.6.4'
 end

data/scientist.gemspec

@@ -18,5 +18,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "minitest", "~> 5.8"
-  gem.add_development_dependency "coveralls", "~> 0.8"
+  gem.add_development_dependency "rake"
 end

data/script/release

@@ -12,10 +12,10 @@ cd $(dirname "$0")/..
 rm -rf scientist-*.gem
 gem build -q scientist.gemspec
 
-# Make sure we're on the master branch.
+# Make sure we're on the main branch.
 
-(git branch --no-color | grep -q '* master') || {
-  echo "Only release from the master branch."
+(git branch --no-color | grep -q '* main') || {
+  echo "Only release from the main branch."
   exit 1
 }
 
@@ -35,4 +35,4 @@ git fetch -t origin
 # Tag it and bag it.
 
 gem push scientist-*.gem && git tag "$tag" &&
-  git push origin master && git push origin "$tag"
+  git push origin main && git push origin "$tag"

data/script/test

@@ -4,9 +4,4 @@
 set -e
 
 cd $(dirname "$0")/..
-  script/bootstrap && bundle exec ruby -I lib \
-    -e 'require "bundler/setup"' \
-    -e 'require "coveralls"; Coveralls.wear!{ add_filter ".bundle" }' \
-    -e 'require "minitest/autorun"' \
-    -e 'require "scientist"' \
-    -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"
+  script/bootstrap && bundle exec rake test

data/test/scientist/experiment_test.rb

@@ -201,6 +201,18 @@ describe Scientist::Experiment do
     assert @ex.published_result.matched?
   end
 
+  it "compares errors with an error comparator block if provided" do
+    @ex.compare_errors { |a, b| a.class == b.class }
+    @ex.use { raise "foo" }
+    @ex.try { raise "bar" }
+
+    resulting_error = assert_raises RuntimeError do
+      @ex.run
+    end
+    assert_equal "foo", resulting_error.message
+    assert @ex.published_result.matched?
+  end
+
   it "knows how to compare two experiments" do
     a = Scientist::Observation.new(@ex, "a") { 1 }
     b = Scientist::Observation.new(@ex, "b") { 2 }
@@ -479,6 +491,27 @@ describe Scientist::Experiment do
       assert_raises(Scientist::Experiment::MismatchError) { runner.call }
     end
 
+    it "can be marshaled" do
+      Fake.raise_on_mismatches = true
+      @ex.before_run { "some block" }
+      @ex.clean { "some block" }
+      @ex.compare_errors { "some block" }
+      @ex.ignore { false }
+      @ex.run_if { "some block" }
+      @ex.try { "candidate" }
+      @ex.use { "control" }
+      @ex.compare { |control, candidate| control == candidate }
+
+      mismatch = nil
+      begin
+        @ex.run
+      rescue Scientist::Experiment::MismatchError => e
+        mismatch = e
+      end
+
+      assert_kind_of(String, Marshal.dump(mismatch))
+    end
+
     describe "#raise_on_mismatches?" do
       it "raises when there is a mismatch if the experiment instance's raise on mismatches is enabled" do
         Fake.raise_on_mismatches = false
@@ -602,6 +635,37 @@ candidate:
     end
   end
 
+  describe "after run block" do
+    it "runs when an experiment is enabled" do
+      control_ok = candidate_ok = false
+      after_result = nil
+      @ex.after_run { |result| after_result = result }
+      @ex.use { control_ok = after_result.nil? }
+      @ex.try { candidate_ok = after_result.nil? }
+
+      @ex.run
+
+      assert after_result, "after_run should have run"
+      assert after_result.matched?, "after_run should be called with the result"
+      assert control_ok, "control should have run before after_run"
+      assert candidate_ok, "candidate should have run before after_run"
+    end
+
+    it "does not run when an experiment is disabled" do
+      after_result = nil
+
+      def @ex.enabled?
+        false
+      end
+      @ex.after_run { |result| after_result = result }
+      @ex.use { "value" }
+      @ex.try { "value" }
+      @ex.run
+
+      refute after_result, "after_run should not have run"
+    end
+  end
+
   describe "testing hooks for extending code" do
     it "allows a user to provide fabricated durations for testing purposes" do
       @ex.use { true }

data/test/scientist/observation_test.rb

@@ -80,7 +80,7 @@ describe Scientist::Observation do
     refute x.equivalent_to?(y)
   end
 
-  FirstErrror = Class.new(StandardError)
+  FirstError = Class.new(StandardError)
   SecondError = Class.new(StandardError)
 
   it "compares exception classes" do
@@ -92,22 +92,46 @@ describe Scientist::Observation do
     refute x.equivalent_to?(y)
   end
 
-  it "compares values using a comparator block" do
+  it "compares values using a comparator proc" do
     a = Scientist::Observation.new("test", @experiment) { 1 }
     b = Scientist::Observation.new("test", @experiment) { "1" }
 
     refute a.equivalent_to?(b)
-    assert a.equivalent_to?(b) { |x, y| x.to_s == y.to_s }
+
+    compare_on_string = -> (x, y) { x.to_s == y.to_s }
+
+    assert a.equivalent_to?(b, compare_on_string)
 
     yielded = []
-    a.equivalent_to?(b) do |x, y|
+    compare_appends = -> (x, y) do
       yielded << x
       yielded << y
       true
     end
+    a.equivalent_to?(b, compare_appends)
+
     assert_equal [a.value, b.value], yielded
   end
 
+  it "compares exceptions using an error comparator proc" do
+    x = Scientist::Observation.new("test", @experiment) { raise FirstError, "error" }
+    y = Scientist::Observation.new("test", @experiment) { raise SecondError, "error" }
+    z = Scientist::Observation.new("test", @experiment) { raise FirstError, "ERROR" }
+
+    refute x.equivalent_to?(z)
+    refute x.equivalent_to?(y)
+
+    compare_on_class = -> (error, other_error) {
+      error.class == other_error.class
+    }
+    compare_on_message = -> (error, other_error) {
+      error.message == other_error.message
+    }
+
+    assert x.equivalent_to?(z, nil, compare_on_class)
+    assert x.equivalent_to?(y, nil, compare_on_message)
+  end
+
   describe "#cleaned_value" do
     it "returns the observation's value by default" do
       a = Scientist::Observation.new("test", @experiment) { 1 }

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+
+$LOAD_PATH.unshift(File.expand_path(File.join(__dir__, "../lib")))
+require "scientist"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scientist
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.4
 platform: ruby
 authors:
 - GitHub Open Source
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-08 00:00:00.000000000 Z
+date: 2023-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -29,19 +29,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.8'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0'
 description: A Ruby library for carefully refactoring critical paths
 email:
 - opensource+scientist@github.com
@@ -53,12 +53,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
 - doc/changelog.md
 - lib/scientist.rb
 - lib/scientist/default.rb
@@ -76,6 +78,7 @@ files:
 - test/scientist/observation_test.rb
 - test/scientist/result_test.rb
 - test/scientist_test.rb
+- test/test_helper.rb
 homepage: https://github.com/github/scientist
 licenses:
 - MIT
@@ -95,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Carefully test, measure, and track refactored code.
@@ -105,3 +108,4 @@ test_files:
 - test/scientist/observation_test.rb
 - test/scientist/result_test.rb
 - test/scientist_test.rb
+- test/test_helper.rb

data/.travis.yml

@@ -1,15 +0,0 @@
----
-language: ruby
-cache: bundler
-script: script/test
-rvm:
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - 2.6.1
-  - truffleruby-head
-before_install: gem install bundler
-addons:
- apt:
-   packages:
-     - libgmp-dev