scientist 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78972aec5f81213b70c0d80841e4608ab6f933c4
-  data.tar.gz: bcdb905245beab4a89e6a2ffe63da983197ddc26
+  metadata.gz: 89ebfbe0cbc1c0de8e8247f59fac21a7ac547e96
+  data.tar.gz: 50b3ee5ed061c80af3ed1e10098acab524d82dc5
 SHA512:
-  metadata.gz: 9428bf617e6c74d85b44b55b5a4e826177cf34dca77f1de80eb25143e13ee78f849873619d1d9362d65d79eb26939686062d4ea34531faa61ff152e70a343e62
-  data.tar.gz: 8a9fdcdf0cb10bc41b65cea0c725450f508890829c60249e0effdc5d09c15ca420f37db31f3968b64d5fe27a76290266b1f90baa1f79cec015c78ddac72e609d
+  metadata.gz: 1c35cd1392cca5da51729d825f9b4b9e58ed3439247141438070ad5137de59600549e12a3eff4a0ed73ee19fa220de4cd66fe37eb2207df5f1a200b7cde532d4
+  data.tar.gz: 80cf84df09f69c2fb3f3f51b6fd72ca3a7d569b0a57c6d5a7dc48d48a597b7add4bbebea149d5dc2bef3aedc9ce5e02897c24285596d9cbbd912f68ec1919e0f

data/README.md

@@ -144,6 +144,25 @@ end
 
 The `widget-permissions` and `widget-destruction` experiments will both have a `:widget` key in their contexts.
 
+### Expensive setup
+
+If an experiment requires expensive setup that should only occur when the experiment is going to be run, define it with the `before_run` method:
+
+```ruby
+# Code under test modifies this in-place. We want to copy it for the
+# candidate code, but only when needed:
+value_for_original_code = big_object
+value_for_new_code      = nil
+
+science "expensive-but-worthwhile" do |e|
+  e.before_run do
+    value_for_new_code = big_object.deep_copy
+  end
+  e.use { original_code(value_for_original_code) }
+  e.try { new_code(value_for_new_code) }
+end
+```
+
 ### Keeping it clean
 
 Sometimes you don't want to store the full value for later analysis. For example, an experiment may return `User` instances, but when researching a mismatch, all you care about is the logins. You can define how to clean these values in an experiment:
@@ -410,7 +429,7 @@ end
 
 ## Hacking
 
-Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are  installed automatically. Science requires Ruby 2.1.
+Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are installed automatically. Science requires Ruby 1.9 or newer.
 
 ## Maintainers
 

data/lib/scientist/experiment.rb

@@ -15,8 +15,33 @@ module Scientist::Experiment
 
   # A mismatch, raised when raise_on_mismatches is enabled.
   class MismatchError < StandardError
+    attr_reader :name, :result
+
     def initialize(name, result)
-      super "#{name}: control #{result.control.inspect}, candidates #{result.candidates.map(&:inspect)}"
+      @name   = name
+      @result = result
+      super "experiment '#{name}' observations mismatched"
+    end
+
+    # The default formatting is nearly unreadable, so make it useful.
+    #
+    # The assumption here is that errors raised in a test environment are
+    # printed out as strings, rather than using #inspect.
+    def to_s
+      super + ":\n" +
+      format_observation(result.control) + "\n" +
+      result.candidates.map { |candidate| format_observation(candidate) }.join("\n") +
+      "\n"
+    end
+
+    def format_observation(observation)
+      observation.name + ":\n" +
+      if observation.raised?
+        observation.exception.inspect.prepend("  ") + "\n" +
+          observation.exception.backtrace.map { |line| line.prepend("    ") }.join("\n")
+      else
+        observation.value.inspect.prepend("  ")
+      end
     end
   end
 
@@ -42,6 +67,16 @@ module Scientist::Experiment
     base.extend RaiseOnMismatch
   end
 
+  # Define a block of code to run before an experiment begins, if the experiment
+  # is enabled.
+  #
+  # The block takes no arguments.
+  #
+  # Returns the configured block.
+  def before_run(&block)
+    @_scientist_before_run = block
+  end
+
   # A Hash of behavior blocks, keyed by String name. Register behavior blocks
   # with the `try` and `use` methods.
   def behaviors
@@ -158,7 +193,13 @@ module Scientist::Experiment
       raise Scientist::BehaviorMissing.new(self, name)
     end
 
-    return block.call unless should_experiment_run?
+    unless should_experiment_run?
+      return block.call
+    end
+
+    if @_scientist_before_run
+      @_scientist_before_run.call
+    end
 
     observations = []
 
@@ -177,15 +218,15 @@ module Scientist::Experiment
       raised :publish, ex
     end
 
-    if control.raised?
-      raise control.exception
-    end
-
     if self.class.raise_on_mismatches? && result.mismatched?
-      raise MismatchError.new(name, result)
+      raise MismatchError.new(self.name, result)
     end
 
-    control.value
+    if control.raised?
+      raise control.exception
+    else
+      control.value
+    end
   end
 
   # Define a block that determines whether or not the experiment should run.

data/lib/scientist/version.rb

@@ -1,3 +1,3 @@
 module Scientist
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/test/scientist/experiment_test.rb

@@ -392,7 +392,112 @@ describe Scientist::Experiment do
       @ex.use { "fine" }
       @ex.try { "not fine" }
 
+      assert_equal "fine", @ex.run
+    end
+
+    it "raises a mismatch error if the control raises and candidate doesn't" do
+      Fake.raise_on_mismatches = true
+      @ex.use { raise "control" }
+      @ex.try { "candidate" }
+      # assert_raises(Scientist::Experiment::MismatchError) { @ex.run }
+      @ex.run
+    end
+
+    it "raises a mismatch error if the candidate raises and the control doesn't" do
+      Fake.raise_on_mismatches = true
+      @ex.use { "control" }
+      @ex.try { raise "candidate" }
+      assert_raises(Scientist::Experiment::MismatchError) { @ex.run }
+    end
+
+    describe "MismatchError" do
+      before do
+        Fake.raise_on_mismatches = true
+        @ex.use { :foo }
+        @ex.try { :bar }
+        begin
+          @ex.run
+        rescue Scientist::Experiment::MismatchError => e
+          @mismatch = e
+        end
+        assert @mismatch
+      end
+
+      it "has the name of the experiment" do
+        assert_equal @ex.name, @mismatch.name
+      end
+
+      it "includes the experiments' results" do
+        assert_equal @ex.published_result, @mismatch.result
+      end
+
+      it "formats nicely as a string" do
+        assert_equal <<-STR, @mismatch.to_s
+experiment 'experiment' observations mismatched:
+control:
+  :foo
+candidate:
+  :bar
+        STR
+      end
+
+      it "includes the backtrace when an observation raises" do
+        mismatch = nil
+        ex = Fake.new
+        ex.use { "value" }
+        ex.try { raise "error" }
+
+        begin
+          ex.run
+        rescue Scientist::Experiment::MismatchError => e
+          mismatch = e
+        end
+
+        # Should look like this:
+        # experiment 'experiment' observations mismatched:
+        # control:
+        #   "value"
+        # candidate:
+        #   #<RuntimeError: error>
+        #     test/scientist/experiment_test.rb:447:in `block (5 levels) in <top (required)>'
+        # ... (more backtrace)
+        lines = mismatch.to_s.split("\n")
+        assert_equal "control:", lines[1]
+        assert_equal "  \"value\"", lines[2]
+        assert_equal "candidate:", lines[3]
+        assert_equal "  #<RuntimeError: error>", lines[4]
+        assert_match %r(    test/scientist/experiment_test.rb:\d+:in `block), lines[5]
+      end
+    end
+  end
+
+  describe "before run block" do
+    it "runs when an experiment is enabled" do
+      control_ok = candidate_ok = false
+      before = false
+      @ex.before_run { before = true }
+      @ex.use { control_ok = before }
+      @ex.try { candidate_ok = before }
+
       @ex.run
+
+      assert before, "before_run should have run"
+      assert control_ok, "control should have run after before_run"
+      assert candidate_ok, "candidate should have run after before_run"
+    end
+
+    it "does not run when an experiment is disabled" do
+      before = false
+
+      def @ex.enabled?
+        false
+      end
+      @ex.before_run { before = true }
+      @ex.use { "value" }
+      @ex.try { "value" }
+      @ex.run
+
+      refute before, "before_run should not have run"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scientist
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - John Barnette
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-30 00:00:00.000000000 Z
+date: 2015-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest