scientist 1.1.2 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a444eefb69bf453db50912ba9a629edd0845f1b2591c36809f4329862188b82
-  data.tar.gz: 28895c2df468c46c10d4b474ee9aa58b448514bb856c6f059112ad48d09bb97b
+  metadata.gz: f03841ae9bccb8d7b979971d5f6a38482f57bbc81af82d0a710479ccc3f16fbb
+  data.tar.gz: c0f81d696918e302462baf1bd918cd25179a99045d48082fb162e1fdb2fa2280
 SHA512:
-  metadata.gz: 40c3e14294c377e70c86e03b50ac121da94b8e79e0b5f14a6844343a4f5b56b6fa340f66caa4671498e303978eea9ebd331ad7633aebd6b5901d970321f71342
-  data.tar.gz: '092d3a37710ea538580205e284b31314eb298b5d10d910d2a36cff0b334ecbbd01c2b9311b35769eec7f308549f954c1a27084306f60232cd18bae24fd59e96d'
+  metadata.gz: f1680e6564a6e83b3dde55f6757f3e7c953a70b3d748aa0e7ca2a54b5a33a4b3a947a8f20aa52790a41e6a66e36627e4e851f627625c0aaf9fd4cf41b94ace8c
+  data.tar.gz: 75ba381053b73e9418ed2ab5608c70c92a55990c938a1c25f4324169ff560a8feea6827ef909e50b9fc3baf5a5eb49f3b7b26ed4e3f9799007b80a71c7543c86

data/.travis.yml

@@ -3,12 +3,11 @@ language: ruby
 cache: bundler
 script: script/test
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.8
-  - 2.2.4
-  - 2.3.0
-  - 2.4.0
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
+  - 2.6.1
+  - truffleruby-head
 before_install: gem install bundler
 addons:
  apt:

data/README.md

@@ -24,9 +24,9 @@ Wrap a `use` block around the code's original behavior, and wrap `try` around th
 
 * It decides whether or not to run the `try` block,
 * Randomizes the order in which `use` and `try` blocks are run,
-* Measures the durations of all behaviors,
+* Measures the durations of all behaviors in seconds,
 * Compares the result of `try` to the result of `use`,
-* Swallows (but records) any exceptions raised in the `try` block, and
+* Swallow and record exceptions raised in the `try` block when overriding `raised`, and
 * Publishes all this information.
 
 The `use` block is called the **control**. The `try` block is called the **candidate**.
@@ -62,7 +62,7 @@ class MyExperiment
 
   attr_accessor :name
 
-  def initialize(name:)
+  def initialize(name)
     @name = name
   end
 
@@ -71,20 +71,21 @@ class MyExperiment
     true
   end
 
+  def raised(operation, error)
+    # see "In a Scientist callback" below
+    p "Operation '#{operation}' failed with error '#{error.inspect}'"
+    super # will re-raise
+  end
+
   def publish(result)
     # see "Publishing results" below
     p result
   end
 end
-
-# replace `Scientist::Default` as the default implementation
-module Scientist::Experiment
-  def self.new(name)
-    MyExperiment.new(name: name)
-  end
-end
 ```
 
+When `Scientist::Experiment` is included in a class, it automatically sets it as the default implementation via `Scientist::Experiment.set_default`. This `set_default` call is is skipped if you include `Scientist::Experiment` in a module.
+
 Now calls to the `science` helper will load instances of `MyExperiment`.
 
 ### Controlling comparison
@@ -206,6 +207,8 @@ class MyExperiment
 end
 ```
 
+Note that the `#clean` method will discard the previous cleaner block if you call it again.  If for some reason you need to access the currently configured cleaner block, `Scientist::Experiment#cleaner` will return the block without further ado.  _(This probably won't come up in normal usage, but comes in handy if you're writing, say, a custom experiment runner that provides default cleaners.)_
+
 ### Ignoring mismatches
 
 During the early stages of an experiment, it's possible that some of your code will always generate a mismatch for reasons you know and understand but haven't yet fixed. Instead of these known cases always showing up as mismatches in your metrics or analysis, you can tell an experiment whether or not to ignore a mismatch using the `ignore` method. You may include more than one block if needed:
@@ -254,7 +257,7 @@ class MyExperiment
 
   attr_accessor :name, :percent_enabled
 
-  def initialize(name:)
+  def initialize(name)
     @name = name
     @percent_enabled = 100
   end
@@ -327,7 +330,7 @@ class MyExperiment
       }
     else
       {
-        # see "Keeping it clean" below
+        # see "Keeping it clean" above
         :value => observation.cleaned_value
       }
     end
@@ -491,6 +494,22 @@ science "various-ways", run: "first-way" do |e|
 end
 ```
 
+#### Providing fake timing data
+
+If you're writing tests that depend on specific timing values, you can provide canned durations using the `fabricate_durations_for_testing_purposes` method, and Scientist will report these in `Scientist::Observation#duration` instead of the actual execution times.
+
+```ruby
+science "absolutely-nothing-suspicious-happening-here" do |e|
+  e.use { ... } # "control"
+  e.try { ... } # "candidate"
+  e.fabricate_durations_for_testing_purposes( "control" => 1.0, "candidate" => 0.5 )
+end
+```
+
+`fabricate_durations_for_testing_purposes` takes a Hash of duration values, keyed by behavior names.  (By default, Scientist uses `"control"` and `"candidate"`, but if you override these as shown in [Trying more than one thing](#trying-more-than-one-thing) or [No control, just candidates](#no-control-just-candidates), use matching names here.)  If a name is not provided, the actual execution time will be reported instead.
+
+_Like `Scientist::Experiment#cleaner`, this probably won't come up in normal usage.  It's here to make it easier to test code that extends Scientist._
+
 ### Without including Scientist
 
 If you need to use Scientist in a place where you aren't able to include the Scientist module, you can call `Scientist.run`:
@@ -504,17 +523,23 @@ end
 
 ## Hacking
 
-Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are installed automatically. Science requires Ruby 1.9 or newer.
+Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are installed automatically. Scientist requires Ruby 2.3 or newer.
+
+## Wrappers
+
+- [RealGeeks/lab_tech](https://github.com/RealGeeks/lab_tech) is a Rails engine for using this library by controlling, storing, and analyzing experiment results with ActiveRecord.
 
 ## Alternatives
 
 - [daylerees/scientist](https://github.com/daylerees/scientist) (PHP)
-- [github/scientist.net](https://github.com/github/scientist.net) (.NET)
+- [scientistproject/scientist.net](https://github.com/scientistproject/Scientist.net) (.NET)
 - [joealcorn/laboratory](https://github.com/joealcorn/laboratory) (Python)
 - [rawls238/Scientist4J](https://github.com/rawls238/Scientist4J) (Java)
 - [tomiaijo/scientist](https://github.com/tomiaijo/scientist) (C++)
 - [trello/scientist](https://github.com/trello/scientist) (node.js)
 - [ziyasal/scientist.js](https://github.com/ziyasal/scientist.js) (node.js, ES6)
+- [TrueWill/tzientist](https://github.com/TrueWill/tzientist) (node.js, TypeScript)
+- [TrueWill/paleontologist](https://github.com/TrueWill/paleontologist) (Deno, TypeScript)
 - [yeller/laboratory](https://github.com/yeller/laboratory) (Clojure)
 - [lancew/Scientist](https://github.com/lancew/Scientist) (Perl 5)
 - [lancew/ScientistP6](https://github.com/lancew/ScientistP6) (Perl 6)
@@ -523,7 +548,9 @@ Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs t
 - [calavera/go-scientist](https://github.com/calavera/go-scientist) (Go)
 - [jelmersnoeck/experiment](https://github.com/jelmersnoeck/experiment) (Go)
 - [spoptchev/scientist](https://github.com/spoptchev/scientist) (Kotlin / Java)
-
+- [junkpiano/scientist](https://github.com/junkpiano/scientist) (Swift)
+- [serverless scientist](http://serverlessscientist.com/) (AWS Lambda)
+- [fightmegg/scientist](https://github.com/fightmegg/scientist) (TypeScript, Browser / Node.js)
 
 ## Maintainers
 

data/doc/changelog.md

@@ -1,5 +1,28 @@
 # Changes
 
+## v1.3.0 (2 April 2019)
+
+- New: Drop support for ruby <2.3
+- Fix: Build new strings instead of modifying frozen ones
+- New: Add an accessor for the configured clean block
+- New: Add a hook to use fabricated durations instead of actual timing data.
+
+## v1.2.0 (5 July 2018)
+
+- New: Use monotonic clock for duration calculations
+- New: Drop support for ruby <2.1 to support monotonic clock
+- New: Run CI on Ruby 2.5
+
+## v1.1.2 (9 May 2018)
+
+- New: Add `raise_with` option to allow for custom mismatch errors to be raised
+
+## v1.1.1 (6 February 2018)
+
+- Fix: default experiment no longer runs all `try` paths
+- New: Add `Scientist.run` module method for running experiments when an included module isn't available
+- New: Add [Noise and error rates](https://github.com/github/scientist#noise-and-error-rates) to `README.md`
+
 ## v1.1.0 (29 August 2017)
 
 - New: [Specify which exception types to rescue](https://github.com/github/scientist#in-candidate-code)

data/lib/scientist/experiment.rb

@@ -9,16 +9,26 @@ module Scientist::Experiment
   # If this is nil, raise_on_mismatches class attribute is used instead.
   attr_accessor :raise_on_mismatches
 
-  # Create a new instance of a class that implements the Scientist::Experiment
-  # interface.
-  #
-  # Override this method directly to change the default implementation.
+  def self.included(base)
+    self.set_default(base) if base.instance_of?(Class)
+    base.extend RaiseOnMismatch
+  end
+
+  # Instantiate a new experiment (using the class given to the .set_default method).
   def self.new(name)
-    Scientist::Default.new(name)
+    (@experiment_klass || Scientist::Default).new(name)
+  end
+
+  # Configure Scientist to use the given class for all future experiments
+  # (must implement the Scientist::Experiment interface).
+  #
+  # Called automatically when new experiments are defined.
+  def self.set_default(klass)
+    @experiment_klass = klass
   end
 
   # A mismatch, raised when raise_on_mismatches is enabled.
-  class MismatchError < StandardError
+  class MismatchError < Exception
     attr_reader :name, :result
 
     def initialize(name, result)
@@ -41,10 +51,10 @@ module Scientist::Experiment
     def format_observation(observation)
       observation.name + ":\n" +
       if observation.raised?
-        observation.exception.inspect.prepend("  ") + "\n" +
-          observation.exception.backtrace.map { |line| line.prepend("    ") }.join("\n")
+        lines = observation.exception.backtrace.map { |line| "    #{line}" }.join("\n")
+        "  #{observation.exception.inspect}" + "\n" + lines
       else
-        observation.cleaned_value.inspect.prepend("  ")
+        "  #{observation.cleaned_value.inspect}"
       end
     end
   end
@@ -67,10 +77,6 @@ module Scientist::Experiment
     end
   end
 
-  def self.included(base)
-    base.extend RaiseOnMismatch
-  end
-
   # Define a block of code to run before an experiment begins, if the experiment
   # is enabled.
   #
@@ -96,6 +102,13 @@ module Scientist::Experiment
     @_scientist_cleaner = block
   end
 
+  # Accessor for the clean block, if one is available.
+  #
+  # Returns the configured block, or nil.
+  def cleaner
+    @_scientist_cleaner
+  end
+
   # Internal: Clean a value with the configured clean block, or return the value
   # if no clean block is configured.
   #
@@ -209,16 +222,7 @@ module Scientist::Experiment
       @_scientist_before_run.call
     end
 
-    observations = []
-
-    behaviors.keys.shuffle.each do |key|
-      block = behaviors[key]
-      observations << Scientist::Observation.new(key, self, &block)
-    end
-
-    control = observations.detect { |o| o.name == name }
-
-    result = Scientist::Result.new self, observations, control
+    result = generate_result(name)
 
     begin
       publish(result)
@@ -234,11 +238,9 @@ module Scientist::Experiment
       end
     end
 
-    if control.raised?
-      raise control.exception
-    else
-      control.value
-    end
+    control = result.control
+    raise control.exception if control.raised?
+    control.value
   end
 
   # Define a block that determines whether or not the experiment should run.
@@ -290,4 +292,24 @@ module Scientist::Experiment
       !!raise_on_mismatches
     end
   end
+
+  # Provide predefined durations to use instead of actual timing data.
+  # This is here solely as a convenience for developers of libraries that extend Scientist.
+  def fabricate_durations_for_testing_purposes(fabricated_durations = {})
+    @_scientist_fabricated_durations = fabricated_durations
+  end
+
+  # Internal: Generate the observations and create the result from those and the control.
+  def generate_result(name)
+    observations = []
+
+    behaviors.keys.shuffle.each do |key|
+      block = behaviors[key]
+      fabricated_duration = @_scientist_fabricated_durations && @_scientist_fabricated_durations[key]
+      observations << Scientist::Observation.new(key, self, fabricated_duration: fabricated_duration, &block)
+    end
+
+    control = observations.detect { |o| o.name == name }
+    Scientist::Result.new(self, observations, control)
+  end
 end

data/lib/scientist/observation.rb

@@ -8,9 +8,6 @@ class Scientist::Observation
   # The experiment this observation is for
   attr_reader :experiment
 
-  # The instant observation began.
-  attr_reader :now
-
   # The String name of the behavior.
   attr_reader :name
 
@@ -23,18 +20,19 @@ class Scientist::Observation
   # The Float seconds elapsed.
   attr_reader :duration
 
-  def initialize(name, experiment, &block)
+  def initialize(name, experiment, fabricated_duration: nil, &block)
     @name       = name
     @experiment = experiment
-    @now        = Time.now
 
+    starting = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) unless fabricated_duration
     begin
       @value = block.call
     rescue *RESCUES => e
       @exception = e
     end
 
-    @duration = (Time.now - @now).to_f
+    @duration = fabricated_duration ||
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - starting
 
     freeze
   end

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "1.1.2"
+  VERSION = "1.6.0"
 end

data/scientist.gemspec

@@ -10,6 +10,8 @@ Gem::Specification.new do |gem|
   gem.homepage      = "https://github.com/github/scientist"
   gem.license       = "MIT"
 
+  gem.required_ruby_version = '>= 2.3'
+
   gem.files         = `git ls-files`.split($/)
   gem.executables   = []
   gem.test_files    = gem.files.grep(/^test/)

data/script/test

@@ -4,7 +4,7 @@
 set -e
 
 cd $(dirname "$0")/..
-  script/bootstrap && ruby -I lib \
+  script/bootstrap && bundle exec ruby -I lib \
     -e 'require "bundler/setup"' \
     -e 'require "coveralls"; Coveralls.wear!{ add_filter ".bundle" }' \
     -e 'require "minitest/autorun"' \

data/test/scientist/experiment_test.rb

@@ -2,6 +2,9 @@ describe Scientist::Experiment do
   class Fake
     include Scientist::Experiment
 
+    # Undo auto-config magic / preserve default behavior of Scientist::Experiment.new
+    Scientist::Experiment.set_default(nil)
+
     def initialize(*args)
     end
 
@@ -28,6 +31,24 @@ describe Scientist::Experiment do
     @ex = Fake.new
   end
 
+  it "sets the default on inclusion" do
+    klass = Class.new do
+      include Scientist::Experiment
+
+      def initialize(name)
+      end
+    end
+
+    assert_kind_of klass, Scientist::Experiment.new("hello")
+
+    Scientist::Experiment.set_default(nil)
+  end
+
+  it "doesn't set the default on inclusion when it's a module" do
+    Module.new { include Scientist::Experiment }
+    assert_kind_of Scientist::Default, Scientist::Experiment.new("hello")
+  end
+
   it "has a default implementation" do
     ex = Scientist::Experiment.new("hello")
     assert_kind_of Scientist::Default, ex
@@ -239,6 +260,13 @@ describe Scientist::Experiment do
     assert_equal 10, @ex.clean_value(10)
   end
 
+  it "provides the clean block when asked for it, in case subclasses wish to override and provide defaults" do
+    assert_nil @ex.cleaner
+    cleaner = ->(value) { value.upcase }
+    @ex.clean(&cleaner)
+    assert_equal cleaner, @ex.cleaner
+  end
+
   it "calls the configured clean block with a value when configured" do
     @ex.clean do |value|
       value.upcase
@@ -437,6 +465,20 @@ describe Scientist::Experiment do
       assert_raises(Scientist::Experiment::MismatchError) { @ex.run }
     end
 
+    it "allows MismatchError to bubble up through bare rescues" do
+      Fake.raise_on_mismatches = true
+      @ex.use { "control" }
+      @ex.try { "candidate" }
+      runner = -> {
+        begin
+          @ex.run
+        rescue
+          # StandardError handled
+        end
+      }
+      assert_raises(Scientist::Experiment::MismatchError) { runner.call }
+    end
+
     describe "#raise_on_mismatches?" do
       it "raises when there is a mismatch if the experiment instance's raise on mismatches is enabled" do
         Fake.raise_on_mismatches = false
@@ -525,7 +567,7 @@ candidate:
         assert_equal "  \"value\"", lines[2]
         assert_equal "candidate:", lines[3]
         assert_equal "  #<RuntimeError: error>", lines[4]
-        assert_match %r(    test/scientist/experiment_test.rb:\d+:in `block), lines[5]
+        assert_match %r(test/scientist/experiment_test.rb:\d+:in `block), lines[5]
       end
     end
   end
@@ -559,4 +601,32 @@ candidate:
       refute before, "before_run should not have run"
     end
   end
+
+  describe "testing hooks for extending code" do
+    it "allows a user to provide fabricated durations for testing purposes" do
+      @ex.use { true }
+      @ex.try { true }
+      @ex.fabricate_durations_for_testing_purposes( "control" => 0.5, "candidate" => 1.0 )
+
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 1.0, cand.duration, 0.01
+    end
+
+    it "returns actual durations if fabricated ones are omitted for some blocks" do
+      @ex.use { true }
+      @ex.try { sleep 0.1; true }
+      @ex.fabricate_durations_for_testing_purposes( "control" => 0.5 )
+
+      @ex.run
+
+      cont = @ex.published_result.control
+      cand = @ex.published_result.candidates.first
+      assert_in_delta 0.5, cont.duration, 0.01
+      assert_in_delta 0.1, cand.duration, 0.01
+    end
+  end
 end

data/test/scientist_test.rb

@@ -56,27 +56,29 @@ describe Scientist do
     obj = Object.new
     obj.extend(Scientist)
 
-    result = obj.science "test", run: "first-way" do |e|
-      experiment = e
+    behaviors_executed = []
 
-      e.try("first-way") { true }
-      e.try("second-way") { true }
+    result = obj.science "test", run: "first-way" do |e|
+      e.try("first-way") { behaviors_executed << "first-way" ; true }
+      e.try("second-way") { behaviors_executed << "second-way" ; true }
     end
 
     assert_equal true, result
+    assert_equal [ "first-way" ], behaviors_executed
   end
 
   it "runs control when there is a nil named test" do
     obj = Object.new
     obj.extend(Scientist)
 
-    result = obj.science "test", nil do |e|
-      experiment = e
+    behaviors_executed = []
 
-      e.use { true }
-      e.try("second-way") { true }
+    result = obj.science "test", nil do |e|
+      e.use { behaviors_executed << "control" ; true }
+      e.try("second-way") { behaviors_executed << "second-way" ; true }
     end
 
     assert_equal true, result
+    assert_equal [ "control" ], behaviors_executed
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scientist
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.6.0
 platform: ruby
 authors:
 - GitHub Open Source
@@ -9,10 +9,10 @@ authors:
 - Rick Bradley
 - Jesse Toth
 - Nathan Witmer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-09 00:00:00.000000000 Z
+date: 2021-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -80,7 +80,7 @@ homepage: https://github.com/github/scientist
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -88,16 +88,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Carefully test, measure, and track refactored code.
 test_files: