scientist 1.6.3 → 1.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53663e0ed4b522336c69b73073aa144a4b6c8178e3fcd716c0e4519a66e308ba
-  data.tar.gz: b60d0c55dce9e1e9a23eb7e6ce4c445bf01340dcfd459d50716d7cd85e405b93
+  metadata.gz: 426765524d161804bb79075cb364ff3101c630da521af30c6f7f79e7e65244a8
+  data.tar.gz: a25d075270905c82d63eff97ae773205c3a1b7f3ddb1fc766ee822dd5ee1edfd
 SHA512:
-  metadata.gz: 6e1baf64882c857abdd3fd02b77785a783a8694854358c78a9da3be213b42ac88e36805ebf7f10f0d87fb14d7808302e1afffd2793c5ffc21addec5326e5f969
-  data.tar.gz: e5a65140ca8704c7443f88404cfb9e971f4a2925d756eed18d7482be74aab3243315fda727dfe8fed13d0732d76ca442a12cb7af200ca6ae3e4de4b0f1bab1a2
+  metadata.gz: c48b5826e463c8c1d59c147509f4ced8e371b022da15da5f8d63b038d2d9ef42945951fbf1901924a8570f24de9c95f4c1b33ca430ed47ac2e243383bde4652c
+  data.tar.gz: 19af496de9ceb3f2d2bdb933656d60830941489ab0702c4753247c606886184a5d1c8c7e25ceda26bc2f8ed5cdf6ec3ee833a74c44ec081a814584e0a5e8b677

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci.yml

@@ -0,0 +1,34 @@
+name: CI
+on: [push, pull_request]
+permissions:
+  actions: none
+  checks: write
+  contents: read
+  deployments: none
+  issues: none
+  packages: none
+  pull-requests: none
+  repository-projects: none
+  security-events: none
+  statuses: write
+jobs:
+  build:
+    name: ruby-${{ matrix.ruby_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby_version:
+          - "2.6"
+          - "2.7"
+          - "3.0"
+          - "3.1"
+          - "3.2"
+          - "3.3"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby_version }}
+          bundler-cache: true
+      - run: bundle exec rake

data/README.md

@@ -1,6 +1,6 @@
 # Scientist!
 
-A Ruby library for carefully refactoring critical paths. [![Build Status](https://travis-ci.org/github/scientist.svg?branch=master)](https://travis-ci.org/github/scientist) [![Coverage Status](https://coveralls.io/repos/github/github/scientist/badge.svg?branch=master)](https://coveralls.io/github/github/scientist?branch=master)
+A Ruby library for carefully refactoring critical paths. [![Build Status](https://github.com/github/scientist/actions/workflows/ci.yml/badge.svg)](https://github.com/github/scientist/actions/workflows/ci.yml)
 
 ## How do I science?
 
@@ -12,7 +12,7 @@ require "scientist"
 class MyWidget
   def allows?(user)
     experiment = Scientist::Default.new "widget-permissions"
-    experiment.use { model.check_user?(user).valid? } # old way
+    experiment.use { model.check_user(user).valid? } # old way
     experiment.try { user.can?(:read, model) } # new way
 
     experiment.run
@@ -24,7 +24,7 @@ Wrap a `use` block around the code's original behavior, and wrap `try` around th
 
 * It decides whether or not to run the `try` block,
 * Randomizes the order in which `use` and `try` blocks are run,
-* Measures the durations of all behaviors in seconds,
+* Measures the wall time and cpu time of all behaviors in seconds,
 * Compares the result of `try` to the result of `use`,
 * Swallow and record exceptions raised in the `try` block when overriding `raised`, and
 * Publishes all this information.
@@ -109,7 +109,7 @@ class MyWidget
 end
 ```
 
-If either the control block or candidate block raises an error, Scientist compares the two observations' classes and messages using `==`. To override this behavior, use `compare_error` to define how to compare observed errors instead:
+If either the control block or candidate block raises an error, Scientist compares the two observations' classes and messages using `==`. To override this behavior, use `compare_errors` to define how to compare observed errors instead:
 
 ```ruby
 class MyWidget
@@ -121,7 +121,7 @@ class MyWidget
       e.try { UserService.slug_from_login login }  # returns String instance or ArgumentError
 
       compare_error_message_and_class = -> (control, candidate) do
-        control.class == candidate.class && 
+        control.class == candidate.class &&
         control.message == candidate.message
       end
 
@@ -129,10 +129,10 @@ class MyWidget
         control.class == ArgumentError &&
         candidate.class == ArgumentError &&
         control.message.start_with?("Input has invalid characters") &&
-        candidate.message.start_with?("Invalid characters in input") 
+        candidate.message.start_with?("Invalid characters in input")
       end
 
-      e.compare_error do |control, candidate|
+      e.compare_errors do |control, candidate|
         compare_error_message_and_class.call(control, candidate) ||
         compare_argument_errors.call(control, candidate)
       end
@@ -241,6 +241,19 @@ end
 
 Note that the `#clean` method will discard the previous cleaner block if you call it again.  If for some reason you need to access the currently configured cleaner block, `Scientist::Experiment#cleaner` will return the block without further ado.  _(This probably won't come up in normal usage, but comes in handy if you're writing, say, a custom experiment runner that provides default cleaners.)_
 
+The `#clean` method will not be used for comparison of the results, so in the following example it is not possible to remove the `#compare` method without the experiment failing:
+
+```ruby
+def user_ids
+  science "user_ids" do
+    e.use { [1,2,3] }
+    e.try { [1,3,2] }
+    e.clean { |value| value.sort }
+    e.compare { |a, b| a.sort == b.sort }
+  end
+end
+```
+
 ### Ignoring mismatches
 
 During the early stages of an experiment, it's possible that some of your code will always generate a mismatch for reasons you know and understand but haven't yet fixed. Instead of these known cases always showing up as mismatches in your metrics or analysis, you can tell an experiment whether or not to ignore a mismatch using the `ignore` method. You may include more than one block if needed:
@@ -260,7 +273,7 @@ def admin?(user)
 end
 ```
 
-The ignore blocks are only called if the *values* don't match. Unless a `compare_error` comparator is defined, two cases are considered mismatches: a) one observation raising an exception and the other not, b) observations raising exceptions with different classes or messages.
+The ignore blocks are only called if the *values* don't match. Unless a `compare_errors` comparator is defined, two cases are considered mismatches: a) one observation raising an exception and the other not, b) observations raising exceptions with different classes or messages.
 
 ### Enabling/disabling experiments
 
@@ -321,11 +334,18 @@ class MyExperiment
 
   def publish(result)
 
+    # Wall time
     # Store the timing for the control value,
     $statsd.timing "science.#{name}.control", result.control.duration
     # for the candidate (only the first, see "Breaking the rules" below,
     $statsd.timing "science.#{name}.candidate", result.candidates.first.duration
 
+    # CPU time
+    # Store the timing for the control value,
+    $statsd.timing "science.cpu.#{name}.control", result.control.cpu_time
+    # for the candidate (only the first, see "Breaking the rules" below,
+    $statsd.timing "science.cpu.#{name}.candidate", result.candidates.first.cpu_time
+
     # and counts for match/ignore/mismatch:
     if result.matched?
       $statsd.increment "science.#{name}.matched"
@@ -530,17 +550,22 @@ end
 
 #### Providing fake timing data
 
-If you're writing tests that depend on specific timing values, you can provide canned durations using the `fabricate_durations_for_testing_purposes` method, and Scientist will report these in `Scientist::Observation#duration` instead of the actual execution times.
+If you're writing tests that depend on specific timing values, you can provide canned durations using the `fabricate_durations_for_testing_purposes` method, and Scientist will report these in `Scientist::Observation#duration` and `Scientist::Observation#cpu_time` instead of the actual execution times.
 
 ```ruby
 science "absolutely-nothing-suspicious-happening-here" do |e|
   e.use { ... } # "control"
   e.try { ... } # "candidate"
-  e.fabricate_durations_for_testing_purposes( "control" => 1.0, "candidate" => 0.5 )
+  e.fabricate_durations_for_testing_purposes({
+    "control" => { "duration" => 1.0, "cpu_time" => 0.9 },
+    "candidate" => { "duration" => 0.5, "cpu_time" => 0.4 }
+  })
 end
 ```
 
-`fabricate_durations_for_testing_purposes` takes a Hash of duration values, keyed by behavior names.  (By default, Scientist uses `"control"` and `"candidate"`, but if you override these as shown in [Trying more than one thing](#trying-more-than-one-thing) or [No control, just candidates](#no-control-just-candidates), use matching names here.)  If a name is not provided, the actual execution time will be reported instead.
+`fabricate_durations_for_testing_purposes` takes a Hash of duration & cpu_time values, keyed by behavior names.  (By default, Scientist uses `"control"` and `"candidate"`, but if you override these as shown in [Trying more than one thing](#trying-more-than-one-thing) or [No control, just candidates](#no-control-just-candidates), use matching names here.)  If a name is not provided, the actual execution time will be reported instead.
+
+We should mention these durations will be used both for the `duration` field and the `cpu_time` field.
 
 _Like `Scientist::Experiment#cleaner`, this probably won't come up in normal usage.  It's here to make it easier to test code that extends Scientist._
 

data/Rakefile

@@ -0,0 +1,7 @@
+require "rake/testtask"
+
+task default: "test"
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/test_helper.rb', 'test/**/*_test.rb']
+end

data/doc/changelog.md

@@ -1,5 +1,49 @@
 # Changes
 
+## v1.6.5 (16 December 2024)
+
+- New: measure CPU time alongside wall time for experiments #275
+
+## v1.6.4 (5 April 2023)
+
+- New: GitHub Actions for CI #171
+- New: add ruby 3.1 support #175
+- Fix: `compare_errors` in docs #178
+- Fix: remove outdated travis configs #179
+- Fix: typos #191
+- New: add support for `after_run` blocks #211
+
+## v1.6.3 (9 December 2021)
+
+- Fix: improve marshaling implementation #169
+
+## v1.6.2 (4 November 2021)
+
+- New: make `MismatchError` marshalable #168
+
+## v1.6.1 (22 October 2021)
+
+- Fix: moving supported ruby versions from <=2.3 to >=2.6 #150
+- Fix: update docs to explain timeout handling #159
+- New: add support for comparing errors #77
+
+## v1.6.0 (8 March 2021)
+
+- Fix: clarify unit for observations #124
+- New: enable support for truffleruby #143
+- Fix: don't default experiment when included in a module #144
+
+## v1.5.0 (8 September 2020)
+
+- Fix: clearer explanation of exception handling #110
+- Fix: remove unused attribute from `Scientist::Observation` #119
+- New: Added internal extension point for generating experinet results #121
+- New: Add `Scientist::Experiment.register` helper #104
+
+## v1.4.0 (20 September 2019)
+
+- New: Make `MismatchError` a base `Exception` #107
+
 ## v1.3.0 (2 April 2019)
 
 - New: Drop support for ruby <2.3

data/lib/scientist/experiment.rb

@@ -87,6 +87,16 @@ module Scientist::Experiment
     @_scientist_before_run = block
   end
 
+  # Define a block of code to run after an experiment completes, if the experiment
+  # is enabled.
+  #
+  # The block takes one argument, the Scientist::Result containing experiment results.
+  #
+  # Returns the configured block.
+  def after_run(&block)
+    @_scientist_after_run = block
+  end
+
   # A Hash of behavior blocks, keyed by String name. Register behavior blocks
   # with the `try` and `use` methods.
   def behaviors
@@ -230,6 +240,10 @@ module Scientist::Experiment
 
     result = generate_result(name)
 
+    if @_scientist_after_run
+      @_scientist_after_run.call(result)
+    end
+
     begin
       publish(result)
     rescue StandardError => ex
@@ -331,7 +345,7 @@ module Scientist::Experiment
     [@name, @result, @raise_on_mismatches]
   end
 
-  def marshal_load
+  def marshal_load(array)
     @name, @result, @raise_on_mismatches = array
   end
 end

data/lib/scientist/observation.rb

@@ -20,19 +20,32 @@ class Scientist::Observation
   # The Float seconds elapsed.
   attr_reader :duration
 
+  # The Float CPU time elapsed, in seconds
+  attr_reader :cpu_time
+
   def initialize(name, experiment, fabricated_duration: nil, &block)
     @name       = name
     @experiment = experiment
 
-    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) unless fabricated_duration
+    start_wall_time, start_cpu_time = capture_times unless fabricated_duration
+
     begin
       @value = block.call
     rescue *RESCUES => e
       @exception = e
     end
 
-    @duration = fabricated_duration ||
-      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - starting
+    if fabricated_duration.is_a?(Hash)
+      @duration = fabricated_duration["duration"]
+      @cpu_time = fabricated_duration["cpu_time"]
+    elsif fabricated_duration
+      @duration = fabricated_duration
+      @cpu_time = 0.0 # setting a default value
+    else
+      end_wall_time, end_cpu_time = capture_times
+      @duration = end_wall_time - start_wall_time
+      @cpu_time = end_cpu_time - start_cpu_time
+    end
 
     freeze
   end
@@ -58,7 +71,7 @@ class Scientist::Observation
   # * The values of the observation are equal (using `==`)
   # * The values of the observations are equal according to a comparison
   #   proc, if given
-  # * The exceptions raised by the obeservations are equal according to the
+  # * The exceptions raised by the observations are equal according to the
   #   error comparison proc, if given.
   # * Both observations raised an exception with the same class and message.
   #
@@ -89,4 +102,13 @@ class Scientist::Observation
   def raised?
     !exception.nil?
   end
+
+  private
+
+  def capture_times
+    [
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second),
+      Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second)
+    ]
+  end
 end

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "1.6.3"
+  VERSION = '1.6.5'
 end

data/scientist.gemspec

@@ -18,5 +18,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "minitest", "~> 5.8"
-  gem.add_development_dependency "coveralls", "~> 0.8"
+  gem.add_development_dependency "rake"
 end

data/script/test

@@ -4,9 +4,4 @@
 set -e
 
 cd $(dirname "$0")/..
-  script/bootstrap && bundle exec ruby -I lib \
-    -e 'require "bundler/setup"' \
-    -e 'require "coveralls"; Coveralls.wear!{ add_filter ".bundle" }' \
-    -e 'require "minitest/autorun"' \
-    -e 'require "scientist"' \
-    -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"
+  script/bootstrap && bundle exec rake test

data/test/scientist/experiment_test.rb

@@ -512,6 +512,10 @@ describe Scientist::Experiment do
       assert_kind_of(String, Marshal.dump(mismatch))
     end
 
+    it "can be marshal loaded" do
+      assert_kind_of(Fake, Marshal.load(Marshal.dump(@ex)))
+    end
+
     describe "#raise_on_mismatches?" do
       it "raises when there is a mismatch if the experiment instance's raise on mismatches is enabled" do
         Fake.raise_on_mismatches = false
@@ -635,8 +639,39 @@ candidate:
     end
   end
 
+  describe "after run block" do
+    it "runs when an experiment is enabled" do
+      control_ok = candidate_ok = false
+      after_result = nil
+      @ex.after_run { |result| after_result = result }
+      @ex.use { control_ok = after_result.nil? }
+      @ex.try { candidate_ok = after_result.nil? }
+
+      @ex.run
+
+      assert after_result, "after_run should have run"
+      assert after_result.matched?, "after_run should be called with the result"
+      assert control_ok, "control should have run before after_run"
+      assert candidate_ok, "candidate should have run before after_run"
+    end
+
+    it "does not run when an experiment is disabled" do
+      after_result = nil
+
+      def @ex.enabled?
+        false
+      end
+      @ex.after_run { |result| after_result = result }
+      @ex.use { "value" }
+      @ex.try { "value" }
+      @ex.run
+
+      refute after_result, "after_run should not have run"
+    end
+  end
+
   describe "testing hooks for extending code" do
-    it "allows a user to provide fabricated durations for testing purposes" do
+    it "allows a user to provide fabricated durations for testing purposes (old version)" do
       @ex.use { true }
       @ex.try { true }
       @ex.fabricate_durations_for_testing_purposes( "control" => 0.5, "candidate" => 1.0 )
@@ -649,7 +684,28 @@ candidate:
       assert_in_delta 1.0, cand.duration, 0.01
     end
 
-    it "returns actual durations if fabricated ones are omitted for some blocks" do
+    it "allows a user to provide fabricated durations for testing purposes (new version)" do
+      @ex.use { true }
+      @ex.try { true }
+      @ex.fabricate_durations_for_testing_purposes({
+        "control" => { "duration" => 0.5, "cpu_time" => 0.4 },
+        "candidate" => { "duration" => 1.0, "cpu_time" => 0.9 }
+      })
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+
+      # Wall Time
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 1.0, cand.duration, 0.01
+
+      # CPU Time
+      assert_equal 0.4, cont.cpu_time
+      assert_equal 0.9, cand.cpu_time
+    end
+
+    it "returns actual durations if fabricated ones are omitted for some blocks (old version)" do
       @ex.use { true }
       @ex.try { sleep 0.1; true }
       @ex.fabricate_durations_for_testing_purposes( "control" => 0.5 )
@@ -661,5 +717,30 @@ candidate:
       assert_in_delta 0.5, cont.duration, 0.01
       assert_in_delta 0.1, cand.duration, 0.01
     end
+
+    it "returns actual durations if fabricated ones are omitted for some blocks (new version)" do
+      @ex.use { true }
+      @ex.try do
+        start_time = Time.now
+        while Time.now - start_time < 0.1
+          # Perform some CPU-intensive work
+          (1..1000).each { |i| i * i }
+        end
+        true
+      end
+      @ex.fabricate_durations_for_testing_purposes({ "control" => { "duration" => 0.5, "cpu_time" => 0.4 }})
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+
+      # Fabricated durations
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 0.4, cont.cpu_time, 0.01
+
+      # Measured durations
+      assert_in_delta 0.1, cand.duration, 0.01
+      assert_in_delta 0.1, cand.cpu_time, 0.01
+    end
   end
 end

data/test/scientist/observation_test.rb

@@ -6,13 +6,18 @@ describe Scientist::Observation do
 
   it "observes and records the execution of a block" do
     ob = Scientist::Observation.new("test", @experiment) do
-      sleep 0.1
+      start_time = Time.now
+      while Time.now - start_time < 0.1
+        # Perform some CPU-intensive work
+        (1..1000).each { |i| i * i }
+      end
       "ret"
     end
 
     assert_equal "ret", ob.value
     refute ob.raised?
     assert_in_delta 0.1, ob.duration, 0.01
+    assert_in_delta 0.1, ob.cpu_time, 0.01
   end
 
   it "stashes exceptions" do

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+
+$LOAD_PATH.unshift(File.expand_path(File.join(__dir__, "../lib")))
+require "scientist"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scientist
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.6.5
 platform: ruby
 authors:
 - GitHub Open Source
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-09 00:00:00.000000000 Z
+date: 2024-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -29,19 +29,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.8'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0'
 description: A Ruby library for carefully refactoring critical paths
 email:
 - opensource+scientist@github.com
@@ -53,12 +53,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
 - doc/changelog.md
 - lib/scientist.rb
 - lib/scientist/default.rb
@@ -76,6 +78,7 @@ files:
 - test/scientist/observation_test.rb
 - test/scientist/result_test.rb
 - test/scientist_test.rb
+- test/test_helper.rb
 homepage: https://github.com/github/scientist
 licenses:
 - MIT
@@ -95,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.5.10
 signing_key:
 specification_version: 4
 summary: Carefully test, measure, and track refactored code.
@@ -105,3 +108,4 @@ test_files:
 - test/scientist/observation_test.rb
 - test/scientist/result_test.rb
 - test/scientist_test.rb
+- test/test_helper.rb

data/.travis.yml

@@ -1,14 +0,0 @@
----
-language: ruby
-cache: bundler
-script: script/test
-rvm:
-  - 2.6.7
-  - 2.7.3
-  - 3.0.1
-  - truffleruby-head
-before_install: gem install bundler
-addons:
- apt:
-   packages:
-     - libgmp-dev