lab_tech 0.1.0

data/spec/examples.txt

@@ -0,0 +1,79 @@
+example_id                                           | status | run_time        |
+---------------------------------------------------- | ------ | --------------- |
+./spec/models/lab_tech/default_cleaner_spec.rb[1:1]  | passed | 0.00112 seconds |
+./spec/models/lab_tech/default_cleaner_spec.rb[1:2]  | passed | 0.0078 seconds  |
+./spec/models/lab_tech/default_cleaner_spec.rb[1:3]  | passed | 0.00039 seconds |
+./spec/models/lab_tech/default_cleaner_spec.rb[1:4]  | passed | 0.00198 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:1:1]   | passed | 0.00335 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:2:1]   | passed | 0.07101 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:2:2]   | passed | 0.01402 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:1] | passed | 0.01324 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:2] | passed | 0.01055 seconds |
+./spec/models/lab_tech/experiment_spec.rb[1:1:2:4:1] | passed | 0.00826 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:1]   | passed | 0.00044 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:2]   | passed | 0.00034 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:3]   | passed | 0.00033 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:4]   | passed | 0.00043 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:5]   | passed | 0.00039 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:6]   | passed | 0.00033 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:7]   | passed | 0.00032 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:8]   | passed | 0.00034 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:9]   | passed | 0.00038 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:1:10]  | passed | 0.00038 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:2:1]   | passed | 0.00039 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:2:2]   | passed | 0.00034 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:2:3]   | passed | 0.00033 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:2:4]   | passed | 0.00031 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:3:1]   | passed | 0.00034 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:3:2]   | passed | 0.00032 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:3:3]   | passed | 0.00031 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:1:3:4]   | passed | 0.0003 seconds  |
+./spec/models/lab_tech/percentile_spec.rb[1:2:1]     | passed | 0.00044 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:2:2]     | passed | 0.00043 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:2:3]     | passed | 0.00035 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:2:4]     | passed | 0.00036 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:2:5]     | passed | 0.00037 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:3:1]     | passed | 0.00047 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:3:2]     | passed | 0.00045 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:3:3]     | passed | 0.00044 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:3:4]     | passed | 0.00043 seconds |
+./spec/models/lab_tech/percentile_spec.rb[1:3:5]     | passed | 0.00045 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:1]         | passed | 0.01435 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:2:1:1]     | passed | 0.00917 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:2:2:1]     | passed | 0.01049 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:2:3:1]     | passed | 0.00984 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:2:3:2]     | passed | 0.00965 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:2:4:1]     | passed | 0.00998 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:3:1]       | passed | 0.00759 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:4:1]       | passed | 0.01068 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:5:1]       | passed | 0.01141 seconds |
+./spec/models/lab_tech/result_spec.rb[1:1:6]         | passed | 0.01769 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:1]          | passed | 0.00127 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:2]          | passed | 0.0004 seconds  |
+./spec/models/lab_tech/speedup_spec.rb[1:3]          | passed | 0.00037 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:4]          | passed | 0.00036 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:5]          | passed | 0.00036 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:6]          | passed | 0.00036 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:7]          | passed | 0.00035 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:8]          | passed | 0.00036 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:9]          | passed | 0.00055 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:10]         | passed | 0.00045 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:11]         | passed | 0.00037 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:12]         | passed | 0.00047 seconds |
+./spec/models/lab_tech/speedup_spec.rb[1:13]         | passed | 0.00048 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:1:1]        | passed | 0.0045 seconds  |
+./spec/models/lab_tech/summary_spec.rb[1:2:1]        | passed | 0.03011 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:3:1]        | passed | 0.01405 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:4:1]        | passed | 0.01349 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:5:1]        | passed | 0.01768 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:5:2]        | passed | 0.01745 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:1:1]      | passed | 0.0281 seconds  |
+./spec/models/lab_tech/summary_spec.rb[1:6:1:2]      | passed | 0.02124 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:2:1]      | passed | 0.02179 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:3:1]      | passed | 0.02039 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:4:1]      | passed | 0.07298 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:4:2]      | passed | 0.05676 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:4:3]      | passed | 0.05425 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:5:1]      | passed | 0.16329 seconds |
+./spec/models/lab_tech/summary_spec.rb[1:6:6:1]      | passed | 0.16302 seconds |
+./spec/models/lab_tech_spec.rb[1:1:1]                | passed | 0.00093 seconds |

data/spec/models/lab_tech/default_cleaner_spec.rb

@@ -0,0 +1,32 @@
+require 'rails_helper'
+
+RSpec.describe LabTech::DefaultCleaner, type: :model do
+  subject(:cleaner) { LabTech::DefaultCleaner }
+
+  def clean(value)
+    cleaner.call(value)
+  end
+
+  it "returns an integer as itself" do
+    expect( clean(42) ).to eq( 42 )
+  end
+
+  it "returns an AR instance as a pair of [ class_name, id ]" do
+    exp = LabTech::Experiment.create(name: "whatever")
+    expect( clean(exp) ).to eq( [ "LabTech::Experiment", exp.id ] )
+  end
+
+  it "returns an array of integers as itself" do
+    expect( clean( [1,2,3] ) ).to eq( [1,2,3] )
+  end
+
+  it "returns an array of AR instances as a hash containing a list of IDs keyed by class name" do
+    e1, e2 = 2.times.map {|i| LabTech::Experiment.create(name: "Experiment #{i}") }
+    expect( clean( [e1, e2] ) ).to eq(
+      {
+        "LabTech::Experiment" => [ e1.id, e2.id ],
+      }
+    )
+  end
+end
+

data/spec/models/lab_tech/experiment_spec.rb

@@ -0,0 +1,110 @@
+require 'rails_helper'
+require SPEC_ROOT.join('support/misc_helpers.rb')
+
+RSpec.describe LabTech::Experiment do
+  around do |example|
+    LabTech.publish_results_in_test_mode do
+      example.run
+    end
+  end
+
+  def wtf
+    puts "", "#" * 100
+    puts "\nExperiments"  ; tp LabTech::Experiment.all
+    puts "\nResults"      ; tp LabTech::Result.all
+    puts "\nObservations" ; tp LabTech::Observation.all
+    puts "", "#" * 100
+  end
+
+  describe ".science" do
+    let!(:experiment) { described_class.create(name: "wibble", percent_enabled: 0) }
+
+    context "by default" do
+      it "runs the .use block (the 'control') but not the .try block (the 'candidate')" do
+        control, candidate = false, false
+        LabTech.science "wibble" do |e|
+          e.use { control   = true }
+          e.try { candidate = true }
+        end
+
+        expect( control   ).to be true
+        expect( candidate ).to be false
+      end
+    end
+
+    context "when the experiment is #enabled?" do
+      before do
+        experiment.update_attribute(:percent_enabled, 100)
+      end
+
+      it "runs the .try block (the 'candidate') when that experiment is #enabled?" do
+        control, candidate = false, false
+        LabTech.science "wibble" do |e|
+          e.use { control   = true }
+          e.try { candidate = true }
+        end
+
+        expect( control   ).to be true
+        expect( candidate ).to be true
+      end
+
+      it "records the results when the experiment is run" do
+        expect( LabTech::Result ).to receive( :record_a_science ).with( experiment, instance_of(Scientist::Result) )
+
+        LabTech.science "wibble" do |e|
+          e.use { :wibble }
+          e.try { :wobble }
+        end
+      end
+
+      describe "value-cleaning behavior" do
+        let(:result) { experiment.results.first }
+
+        specify "if a #clean block IS provided, it is used" do
+          LabTech.science "wibble" do |e|
+            e.use { :control }
+            e.try { :candidate }
+            e.clean { |value| value.to_s.upcase }
+          end
+
+          result = experiment.results.first
+          expect( result ).to be_kind_of( LabTech::Result )
+
+          expect( result.control          .value ).to eq( "CONTROL" )
+          expect( result.candidates.first .value ).to eq( "CANDIDATE" )
+        end
+
+        specify "if a #clean block IS NOT provided, DefaultCleaner is used" do
+          default = LabTech::DefaultCleaner
+          expect( default ).to receive( :call ).with( :control   ).and_return( "Yes indeedily!" )
+          expect( default ).to receive( :call ).with( :candidate ).and_return( "You suck-diddly-uck, Flanders!" )
+
+          LabTech.science "wibble" do |e|
+            e.use { :control }
+            e.try { :candidate }
+          end
+
+          result = experiment.results.first
+          expect( result.control          .value ).to eq( "Yes indeedily!" )
+          expect( result.candidates.first .value ).to eq( "You suck-diddly-uck, Flanders!" )
+        end
+      end
+
+      describe "result counts" do
+        specify "when results are equivalent" do
+          LabTech.science "wibble" do |e|
+            e.use { :wibble }
+            e.try { :wibble }
+          end
+
+          experiment.reload
+          aggregate_failures do
+            expect( experiment.equivalent_count  ).to eq( 1 )
+            expect( experiment.timed_out_count   ).to eq( 0 )
+            expect( experiment.other_error_count ).to eq( 0 )
+          end
+        end
+    end
+    end
+  end
+end

data/spec/models/lab_tech/percentile_spec.rb

@@ -0,0 +1,85 @@
+require 'rails_helper'
+
+RSpec.describe LabTech::Percentile do
+  # Just keep walking, Mr. Phippen, and nobody gets hurt. ;P
+  def self.expect_percentile( percentile, expected_value )
+    specify "percentile(#{percentile}) should be #{expected_value}" do
+      expect( LabTech::Percentile.call(percentile, array) ).to eq( expected_value )
+    end
+  end
+
+  def self.expect_percentiles(percentiles_to_values = {})
+    percentiles_to_values.each do |percentile, value|
+      expect_percentile percentile, value
+    end
+  end
+
+  describe "examples I swiped from Wikipedia" do
+    # Specifically: https://en.wikipedia.org/wiki/Percentile#The_nearest-rank_method
+    context "for a 5-item array" do
+      subject(:array) { [ 15, 20, 35, 40, 50 ] }
+
+      expect_percentiles({
+        0   => 15,
+        20  => 15,
+        21  => 20,
+        40  => 20,
+        41  => 35,
+        60  => 35,
+        61  => 40,
+        80  => 40,
+        81  => 50,
+        100 => 50,
+      })
+
+    end
+
+    context "for a 10-item array" do
+      subject(:array) { [ 3, 6, 7, 8, 8, 10, 13, 15, 16, 20 ] }
+
+      expect_percentiles({
+        25  => 7,
+        50  => 8,
+        75  => 15,
+        100 => 20,
+      })
+    end
+
+    context "for an 11-item array" do
+      subject(:array) { [ 3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20 ] }
+
+      expect_percentiles({
+        25  => 7,
+        50  => 9,
+        75  => 15,
+        100 => 20,
+      })
+
+    end
+  end
+
+  context "for a 100-item array" do
+    subject(:array) { (1..100).to_a }
+
+    expect_percentiles({
+      0   =>   1,
+      1   =>   1,
+      2   =>   2,
+      99  =>  99,
+      100 => 100,
+    })
+  end
+
+  context "for a 1000-item array" do
+    subject(:array) { (1..1000).to_a }
+
+    expect_percentiles({
+      0   => 1,
+      1   => 10,
+      50  => 500,
+      99  => 990,
+      100 => 1000,
+    })
+
+  end
+end

data/spec/models/lab_tech/result_spec.rb

@@ -0,0 +1,198 @@
+require 'rails_helper'
+require SPEC_ROOT.join('support/misc_helpers.rb')
+
+RSpec.describe LabTech::Result, type: :model do
+  let!(:experiment) { LabTech::Experiment.create(name: "wibble", percent_enabled: 100) }
+
+  around do |example|
+    LabTech.publish_results_in_test_mode do
+      example.run
+    end
+  end
+
+  def wibble_wobble!( fabricated_durations: {} )
+    LabTech.science "wibble" do |e|
+      e.use { :wibble }
+      e.try { :wobble }
+
+      # As of 1.3.0, Scientist allows you to provide fake timing data :)
+      e.fabricate_durations_for_testing_purposes( fabricated_durations )
+    end
+  end
+
+  describe ".record_a_science" do
+    let(:result)    { LabTech::Result.last }
+    let(:control)   { result.control }
+    let(:candidate) { result.candidates.first }
+
+    it "creates records for the result and both observations" do
+      expect { wibble_wobble! } \
+        .to  change { LabTech::Result      .count }.by( 1 )
+        .and change { LabTech::Observation .count }.by( 2 )
+
+      aggregate_failures do
+        expect( result.equivalent   ).to be false
+        expect( result.raised_error ).to be false
+
+        expect( result.control.value          ).to eq( :wibble )
+        expect( result.candidates.first.value ).to eq( :wobble )
+      end
+    end
+
+    describe "timing data" do
+      context "when one behavior takes zero time" do
+        let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.0 } }
+
+        specify "the saved records contain timing data (durations, delta, but no speedup)" do
+          wibble_wobble! fabricated_durations: fabricated_durations
+
+          expect( control.duration          ).to eq(  0.5 )
+          expect( candidate.duration        ).to eq(  0.0 )
+          expect( result.control_duration   ).to eq(  0.5 )
+          expect( result.candidate_duration ).to eq(  0.0 )
+          expect( result.time_delta         ).to eq(  0.5 )
+          expect( result.speedup_factor     ).to be nil
+        end
+      end
+
+      context "when both behaviors take zero time" do
+        let(:fabricated_durations) { { "control" => 0.0, "candidate" => 0.0 } }
+
+        specify "the saved records contain timing data (durations, delta, but no speedup)" do
+          wibble_wobble! fabricated_durations: { "control" => 0.0, "candidate" => 0.0 }
+
+          expect( control.duration          ).to eq( 0.0 )
+          expect( candidate.duration        ).to eq( 0.0 )
+          expect( result.control_duration   ).to eq( 0.0 )
+          expect( result.candidate_duration ).to eq( 0.0 )
+          expect( result.time_delta         ).to eq( 0.0 )
+          expect( result.speedup_factor     ).to be nil
+        end
+      end
+
+      context "when both behaviors take exactly the same time" do
+        let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.5 } }
+
+        specify "the saved records contain timing data (durations, delta, speedup)" do
+          wibble_wobble! fabricated_durations: fabricated_durations
+
+          expect( control.duration          ).to eq( 0.5 )
+          expect( candidate.duration        ).to eq( 0.5 )
+          expect( result.control_duration   ).to eq( 0.5 )
+          expect( result.candidate_duration ).to eq( 0.5 )
+          expect( result.time_delta         ).to eq( 0.0 )
+          expect( result.speedup_factor     ).to eq( 0.0 )
+        end
+
+        specify "the result has a Speedup object" do
+          wibble_wobble! fabricated_durations: fabricated_durations
+          speedup = result.speedup
+
+          expect( speedup ).to be_kind_of( LabTech::Speedup )
+
+          expect( speedup.time   ).to eq( 0.0 )
+          expect( speedup.factor ).to eq( 0.0 )
+        end
+      end
+
+      context "when one behavior takes twice as long as the other" do
+        let(:fabricated_durations) { { "control" => 0.5, "candidate" => 1.0 } }
+
+        specify "the saved records contain timing data (durations, delta, speedup)" do
+          wibble_wobble! fabricated_durations: fabricated_durations
+
+          expect( control.duration      ).to eq(  0.5 )
+          expect( candidate.duration    ).to eq(  1.0 )
+          expect( result.time_delta     ).to eq( -0.5 )
+          expect( result.speedup_factor ).to eq( -2.0 )
+        end
+      end
+    end
+
+    context "when a comparator is provided" do
+      before do
+        LabTech.science "wibble" do |e|
+          e.use { :wibble }
+          e.try { :WIBBLE }
+          e.compare { |control, candidate| control.to_s.upcase == candidate.to_s.upcase }
+        end
+      end
+
+      specify "we use it to check equivalency" do
+        expect( result ).to be_equivalent
+      end
+    end
+
+    context "when the candidate raises an exception" do
+      before do
+        LabTech.science "wibble" do |e|
+          e.use { :wibble }
+          e.try { raise "nope" }
+        end
+      end
+
+      specify "we don't asplode" do
+        aggregate_failures do
+          expect( result   .raised_error? ).to be true
+          expect( control  .raised_error? ).to be false
+          expect( candidate.raised_error? ).to be true
+
+          expect( result   .timed_out? ).to be false
+          expect( control  .timed_out? ).to be false
+          expect( candidate.timed_out? ).to be false
+
+          expect( candidate.exception_class     ).to eq( "RuntimeError" )
+          expect( candidate.exception_message   ).to eq( "nope" )
+          expect( candidate.exception_backtrace ).to be_present
+        end
+      end
+    end
+
+    context "when the exception raised is a Timeout::Error" do
+      before do
+        LabTech.science "wibble" do |e|
+          e.use { :wibble }
+          e.try { raise Timeout::Error, "nope" }
+        end
+      end
+
+      specify "we mark the result as :timed_out AND :raised_error" do
+        aggregate_failures do
+          expect( result   .raised_error? ).to be true
+          expect( control  .raised_error? ).to be false
+          expect( candidate.raised_error? ).to be true
+
+          expect( result   .timed_out? ).to be true
+          expect( control  .timed_out? ).to be false
+          expect( candidate.timed_out? ).to be true
+
+          expect( candidate.exception_class     ).to eq( "Timeout::Error" )
+          expect( candidate.exception_message   ).to eq( "nope" )
+          expect( candidate.exception_backtrace ).to be_present
+        end
+
+        expect( LabTech::Result.timed_out ).to include( result )
+      end
+    end
+
+    specify "Results that time out are *not* also counted as mismatches" do
+      LabTech.science "wibble" do |e|
+        e.use { :wibble }
+        e.try { raise Timeout::Error, "nope" }
+      end
+
+      aggregate_failures do
+        expect( result.equivalent?   ).to be false
+        expect( result.timed_out?    ).to be true
+        expect( result.raised_error? ).to be true
+
+        expect( described_class.correct     ).to_not include( result )
+        expect( described_class.mismatched  ).to_not include( result )
+        expect( described_class.timed_out   ).to     include( result )
+        expect( described_class.other_error ).to_not include( result )
+      end
+
+      expect( LabTech::Result.timed_out ).to include( result )
+    end
+  end
+end