scientist 0.0.0 → 0.0.1

data/lib/scientist/experiment.rb

@@ -0,0 +1,233 @@
+# This mixin provides shared behavior for experiments. Includers must implement
+# `enabled?` and `publish(result)`.
+#
+# Override Scientist::Experiment.new to set your own class which includes and
+# implements Scientist::Experiment's interface.
+module Scientist::Experiment
+
+  # Create a new instance of a class that implements the Scientist::Experiment
+  # interface.
+  #
+  # Override this method directly to change the default implementation.
+  def self.new(name)
+    Scientist::Default.new(name)
+  end
+
+  # A mismatch, raised when raise_on_mismatches is enabled.
+  class MismatchError < StandardError
+    def initialize(name, result)
+      super "#{name}: control #{result.control.inspect}, candidates #{result.candidates.map(&:inspect)}"
+    end
+  end
+
+  module RaiseOnMismatch
+    # Set this flag to raise on experiment mismatches.
+    #
+    # This causes all science mismatches to raise a MismatchError. This is
+    # intended for test environments and should not be enabled in a production
+    # environment.
+    #
+    # bool - true/false - whether to raise when the control and candidate mismatch.
+    def raise_on_mismatches=(bool)
+      @raise_on_mismatches = bool
+    end
+
+    # Whether or not to raise a mismatch error when a mismatch occurs.
+    def raise_on_mismatches?
+      @raise_on_mismatches
+    end
+  end
+
+  def self.included(base)
+    base.extend RaiseOnMismatch
+  end
+
+  # A Hash of behavior blocks, keyed by String name. Register behavior blocks
+  # with the `try` and `use` methods.
+  def behaviors
+    @_scientist_behaviors ||= {}
+  end
+
+  # A block to clean an observed value for publishing or storing.
+  #
+  # The block takes one argument, the observed value which will be cleaned.
+  #
+  # Returns the configured block.
+  def clean(&block)
+    @_scientist_cleaner = block
+  end
+
+  # Internal: Clean a value with the configured clean block, or return the value
+  # if no clean block is configured.
+  #
+  # Rescues and reports exceptions in the clean block if they occur.
+  def clean_value(value)
+    if @_scientist_cleaner
+      @_scientist_cleaner.call value
+    else
+      value
+    end
+  rescue StandardError => ex
+    raised :clean, ex
+    value
+  end
+
+  # A block which compares two experimental values.
+  #
+  # The block must take two arguments, the control value and a candidate value,
+  # and return true or false.
+  #
+  # Returns the block.
+  def compare(*args, &block)
+    @_scientist_comparator = block
+  end
+
+  # A Symbol-keyed Hash of extra experiment data.
+  def context(context = nil)
+    @_scientist_context ||= {}
+    @_scientist_context.merge!(context) if !context.nil?
+    @_scientist_context
+  end
+
+  # Configure this experiment to ignore an observation with the given block.
+  #
+  # The block takes two arguments, the control observation and the candidate
+  # observation which didn't match the control. If the block returns true, the
+  # mismatch is disregarded.
+  #
+  # This can be called more than once with different blocks to use.
+  def ignore(&block)
+    @_scientist_ignores ||= []
+    @_scientist_ignores << block
+  end
+
+  # Internal: ignore a mismatched observation?
+  #
+  # Iterates through the configured ignore blocks and calls each of them with
+  # the given control and mismatched candidate observations.
+  #
+  # Returns true or false.
+  def ignore_mismatched_observation?(control, candidate)
+    return false unless @_scientist_ignores
+    @_scientist_ignores.any? do |ignore|
+      begin
+        ignore.call control.value, candidate.value
+      rescue StandardError => ex
+        raised :ignore, ex
+        false
+      end
+    end
+  end
+
+  # The String name of this experiment. Default is "experiment". See
+  # Scientist::Default for an example of how to override this default.
+  def name
+    "experiment"
+  end
+
+  # Internal: compare two observations, using the configured compare block if present.
+  def observations_are_equivalent?(a, b)
+    if @_scientist_comparator
+      a.equivalent_to?(b, &@_scientist_comparator)
+    else
+      a.equivalent_to? b
+    end
+  rescue StandardError => ex
+    raised :compare, ex
+    false
+  end
+
+  # Called when an exception is raised while running an internal operation,
+  # like :publish. Override this method to track these exceptions. The
+  # default implementation re-raises the exception.
+  def raised(operation, error)
+    raise error
+  end
+
+  # Internal: Run all the behaviors for this experiment, observing each and
+  # publishing the results. Return the result of the named behavior, default
+  # "control".
+  def run(name = nil)
+    behaviors.freeze
+    context.freeze
+
+    name = (name || "control").to_s
+    block = behaviors[name]
+
+    if block.nil?
+      raise Scientist::BehaviorMissing.new(self, name)
+    end
+
+    return block.call unless should_experiment_run?
+
+    observations = []
+
+    behaviors.keys.shuffle.each do |key|
+      block = behaviors[key]
+      observations << Scientist::Observation.new(key, self, &block)
+    end
+
+    control = observations.detect { |o| o.name == name }
+
+    result = Scientist::Result.new self,
+      observations: observations,
+      control: control
+
+    begin
+      publish(result)
+    rescue StandardError => ex
+      raised :publish, ex
+    end
+
+    if control.raised?
+      raise control.exception
+    end
+
+    if self.class.raise_on_mismatches? && result.mismatched?
+      raise MismatchError.new(name, result)
+    end
+
+    control.value
+  end
+
+  # Define a block that determines whether or not the experiment should run.
+  def run_if(&block)
+    @_scientist_run_if_block = block
+  end
+
+  # Internal: does a run_if block allow the experiment to run?
+  #
+  # Rescues and reports exceptions in a run_if block if they occur.
+  def run_if_block_allows?
+    (@_scientist_run_if_block ? @_scientist_run_if_block.call : true)
+  rescue StandardError => ex
+    raised :run_if, ex
+    return false
+  end
+
+  # Internal: determine whether or not an experiment should run.
+  #
+  # Rescues and reports exceptions in the enabled method if they occur.
+  def should_experiment_run?
+    behaviors.size > 1 && enabled? && run_if_block_allows?
+  rescue StandardError => ex
+    raised :enabled, ex
+    return false
+  end
+
+  # Register a named behavior for this experiment, default "candidate".
+  def try(name = nil, &block)
+    name = (name || "candidate").to_s
+
+    if behaviors.include?(name)
+      raise Scientist::BehaviorNotUnique.new(self, name)
+    end
+
+    behaviors[name] = block
+  end
+
+  # Register the control behavior for this experiment.
+  def use(&block)
+    try "control", &block
+  end
+end

data/lib/scientist/observation.rb

@@ -0,0 +1,92 @@
+# What happened when this named behavior was executed? Immutable.
+class Scientist::Observation
+
+  # The experiment this observation is for
+  attr_reader :experiment
+
+  # The instant observation began.
+  attr_reader :now
+
+  # The String name of the behavior.
+  attr_reader :name
+
+  # The value returned, if any.
+  attr_reader :value
+
+  # The raised exception, if any.
+  attr_reader :exception
+
+  # The Float seconds elapsed.
+  attr_reader :duration
+
+  def initialize(name, experiment, &block)
+    @name       = name
+    @experiment = experiment
+    @now        = Time.now
+
+    begin
+      @value = block.call
+    rescue Object => e
+      @exception = e
+    end
+
+    @duration = (Time.now - @now).to_f
+
+    freeze
+  end
+
+  # Return a cleaned value suitable for publishing. Uses the experiment's
+  # defined cleaner block to clean the observed value.
+  def cleaned_value
+    if value
+      experiment.clean_value value
+    end
+  end
+
+  # Is this observation equivalent to another?
+  #
+  # other      - the other Observation in question
+  # comparator - an optional comparison block. This observation's value and the
+  #              other observation's value are yielded to this to determine
+  #              their equivalency. Block should return true/false.
+  #
+  # Returns true if:
+  #
+  # * The values of the observation are equal (using `==`)
+  # * The values of the observations are equal according to a comparison
+  #   block, if given
+  # * Both observations raised an exception with the same class and message.
+  #
+  # Returns false otherwise.
+  def equivalent_to?(other, &comparator)
+    return false unless other.is_a?(Scientist::Observation)
+
+    values_are_equal = false
+    both_raised      = other.raised? && raised?
+    neither_raised   = !other.raised? && !raised?
+
+    if neither_raised
+      if block_given?
+        values_are_equal = yield value, other.value
+      else
+        values_are_equal = value == other.value
+      end
+    end
+
+    exceptions_are_equivalent = # backtraces will differ, natch
+      both_raised &&
+        other.exception.class == exception.class &&
+          other.exception.message == exception.message
+
+    (neither_raised && values_are_equal) ||
+      (both_raised && exceptions_are_equivalent)
+  end
+
+  def hash
+    [value, exception, self.class].compact.map(&:hash).inject(:^)
+  end
+
+  def raised?
+    !exception.nil?
+  end
+end

data/lib/scientist/result.rb

@@ -0,0 +1,77 @@
+# The immutable result of running an experiment.
+class Scientist::Result
+
+  # An Array of candidate Observations.
+  attr_reader :candidates
+
+  # The control Observation to which the rest are compared.
+  attr_reader :control
+
+  # An Experiment.
+  attr_reader :experiment
+
+  # An Array of observations which didn't match the control, but were ignored.
+  attr_reader :ignored
+
+  # An Array of observations which didn't match the control.
+  attr_reader :mismatched
+
+  # An Array of Observations in execution order.
+  attr_reader :observations
+
+  # Internal: Create a new result.
+  #
+  # experiment    - the Experiment this result is for
+  # observations: - an Array of Observations, in execution order
+  # control:      - the control Observation
+  #
+  def initialize(experiment, observations:, control:)
+    @experiment   = experiment
+    @observations = observations
+    @control      = control
+    @candidates   = observations - [control]
+    evaluate_candidates
+
+    freeze
+  end
+
+  # Public: the experiment's context
+  def context
+    experiment.context
+  end
+
+  # Public: the name of the experiment
+  def experiment_name
+    experiment.name
+  end
+
+  # Public: was the result a match between all behaviors?
+  def matched?
+    mismatched.empty?
+  end
+
+  # Public: were there mismatches in the behaviors?
+  def mismatched?
+    mismatched.any?
+  end
+
+  # Public: were there any ignored mismatches?
+  def ignored?
+    ignored.any?
+  end
+
+  # Internal: evaluate the candidates to find mismatched and ignored results
+  #
+  # Sets @ignored and @mismatched with the ignored and mismatched candidates.
+  def evaluate_candidates
+    mismatched = candidates.reject do |candidate|
+      experiment.observations_are_equivalent?(control, candidate)
+    end
+
+    @ignored = mismatched.select do |candidate|
+      experiment.ignore_mismatched_observation? control, candidate
+    end
+
+    @mismatched = mismatched - @ignored
+  end
+end

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "0.0.0"
+  VERSION = "0.0.1"
 end

data/lib/scientist.rb

@@ -1,4 +1,41 @@
-require "scientist/version"
-
+# Include this module into any class which requires science experiments in its
+# methods. Provides the `science` and `default_scientist_context` methods for
+# defining and running experiments.
+#
+# If you need to run science on class methods, extend this module instead.
 module Scientist
+  # Define and run a science experiment.
+  #
+  # name - a String name for this experiment.
+  # run: - optional argument for which named test to run instead of "control".
+  #
+  # Yields an object which implements the Scientist::Experiment interface.
+  # See `Scientist::Experiment.new` for how this is defined.
+  #
+  # Returns the calculated value of the control experiment, or raises if an
+  # exception was raised.
+  def science(name, run: nil)
+    experiment = Experiment.new(name)
+    experiment.context(default_scientist_context)
+
+    yield experiment
+
+    experiment.run(run)
+  end
+
+  # Public: the default context data for an experiment created and run via the
+  # `science` helper method. Override this in any class that includes Scientist
+  # to define your own behavior.
+  #
+  # Returns a Hash.
+  def default_scientist_context
+    {}
+  end
 end
+
+require "scientist/default"
+require "scientist/errors"
+require "scientist/experiment"
+require "scientist/observation"
+require "scientist/result"
+require "scientist/version"

data/scientist.gemspec

@@ -1,7 +1,8 @@
-load "lib/scientist/version.rb"
+$: << "lib" and require "scientist/version"
 
 Gem::Specification.new do |gem|
   gem.name          = "scientist"
+  gem.description   = "A Ruby library for carefully refactoring critical paths"
   gem.version       = Scientist::VERSION
   gem.authors       = ["John Barnette", "Rick Bradley"]
   gem.email         = ["jbarnette@github.com", "rick@github.com"]
@@ -9,11 +10,12 @@ Gem::Specification.new do |gem|
   gem.homepage      = "https://github.com/github/scientist"
   gem.license       = "MIT"
 
+  gem.required_ruby_version = ">= 2.1.0"
+
   gem.files         = `git ls-files`.split($/)
   gem.executables   = []
   gem.test_files    = gem.files.grep(/^test/)
   gem.require_paths = ["lib"]
 
-  gem.add_development_dependency "minitest", "~> 5.2.2"
-  gem.add_development_dependency "mocha",    "~> 1.0.0"
+  gem.add_development_dependency "minitest", "~> 5.2"
 end

data/script/test

@@ -4,5 +4,8 @@
 set -e
 
 cd $(dirname "$0")/..
-  script/bootstrap && ruby -I lib -e 'require "bundler/setup"' \
+  script/bootstrap && ruby -I lib \
+    -e 'require "bundler/setup"' \
+    -e 'require "minitest/autorun"' \
+    -e 'require "scientist"' \
     -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"

data/test/scientist/default_test.rb

@@ -0,0 +1,23 @@
+describe Scientist::Default do
+  before do
+    @ex = Scientist::Default.new "default"
+  end
+
+  it "is always enabled" do
+    assert @ex.enabled?
+  end
+
+  it "noops publish" do
+    assert_nil @ex.publish("data")
+  end
+
+  it "is an experiment" do
+    assert Scientist::Default < Scientist::Experiment
+  end
+
+  it "reraises when an internal action raises" do
+     assert_raises RuntimeError do
+       @ex.raised :publish, RuntimeError.new("kaboom")
+     end
+  end
+end