lab_tech 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.md +323 -0
  4. data/Rakefile +30 -0
  5. data/app/models/lab_tech/application_record.rb +5 -0
  6. data/app/models/lab_tech/default_cleaner.rb +87 -0
  7. data/app/models/lab_tech/experiment.rb +190 -0
  8. data/app/models/lab_tech/observation.rb +40 -0
  9. data/app/models/lab_tech/percentile.rb +41 -0
  10. data/app/models/lab_tech/result.rb +130 -0
  11. data/app/models/lab_tech/speedup.rb +65 -0
  12. data/app/models/lab_tech/summary.rb +183 -0
  13. data/config/routes.rb +2 -0
  14. data/db/migrate/20190815192130_create_experiment_tables.rb +50 -0
  15. data/lib/lab_tech.rb +176 -0
  16. data/lib/lab_tech/engine.rb +6 -0
  17. data/lib/lab_tech/version.rb +3 -0
  18. data/lib/tasks/lab_tech_tasks.rake +4 -0
  19. data/spec/dummy/Rakefile +6 -0
  20. data/spec/dummy/app/assets/config/manifest.js +1 -0
  21. data/spec/dummy/app/assets/javascripts/application.js +14 -0
  22. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  23. data/spec/dummy/app/controllers/application_controller.rb +2 -0
  24. data/spec/dummy/app/jobs/application_job.rb +2 -0
  25. data/spec/dummy/app/models/application_record.rb +3 -0
  26. data/spec/dummy/bin/bundle +3 -0
  27. data/spec/dummy/bin/rails +4 -0
  28. data/spec/dummy/bin/rake +4 -0
  29. data/spec/dummy/bin/setup +33 -0
  30. data/spec/dummy/bin/update +28 -0
  31. data/spec/dummy/config.ru +5 -0
  32. data/spec/dummy/config/application.rb +35 -0
  33. data/spec/dummy/config/boot.rb +5 -0
  34. data/spec/dummy/config/database.yml +25 -0
  35. data/spec/dummy/config/environment.rb +5 -0
  36. data/spec/dummy/config/environments/development.rb +46 -0
  37. data/spec/dummy/config/environments/production.rb +71 -0
  38. data/spec/dummy/config/environments/test.rb +36 -0
  39. data/spec/dummy/config/initializers/application_controller_renderer.rb +8 -0
  40. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  41. data/spec/dummy/config/initializers/cors.rb +16 -0
  42. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  43. data/spec/dummy/config/initializers/inflections.rb +16 -0
  44. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  45. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  46. data/spec/dummy/config/locales/en.yml +33 -0
  47. data/spec/dummy/config/puma.rb +34 -0
  48. data/spec/dummy/config/routes.rb +3 -0
  49. data/spec/dummy/config/spring.rb +6 -0
  50. data/spec/dummy/db/schema.rb +52 -0
  51. data/spec/dummy/db/test.sqlite3 +0 -0
  52. data/spec/dummy/log/development.log +0 -0
  53. data/spec/dummy/log/test.log +1519 -0
  54. data/spec/examples.txt +79 -0
  55. data/spec/models/lab_tech/default_cleaner_spec.rb +32 -0
  56. data/spec/models/lab_tech/experiment_spec.rb +110 -0
  57. data/spec/models/lab_tech/percentile_spec.rb +85 -0
  58. data/spec/models/lab_tech/result_spec.rb +198 -0
  59. data/spec/models/lab_tech/speedup_spec.rb +133 -0
  60. data/spec/models/lab_tech/summary_spec.rb +325 -0
  61. data/spec/models/lab_tech_spec.rb +23 -0
  62. data/spec/rails_helper.rb +62 -0
  63. data/spec/spec_helper.rb +98 -0
  64. data/spec/support/misc_helpers.rb +7 -0
  65. metadata +238 -0
@@ -0,0 +1,79 @@
1
+ example_id | status | run_time |
2
+ ---------------------------------------------------- | ------ | --------------- |
3
+ ./spec/models/lab_tech/default_cleaner_spec.rb[1:1] | passed | 0.00112 seconds |
4
+ ./spec/models/lab_tech/default_cleaner_spec.rb[1:2] | passed | 0.0078 seconds |
5
+ ./spec/models/lab_tech/default_cleaner_spec.rb[1:3] | passed | 0.00039 seconds |
6
+ ./spec/models/lab_tech/default_cleaner_spec.rb[1:4] | passed | 0.00198 seconds |
7
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:1:1] | passed | 0.00335 seconds |
8
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:2:1] | passed | 0.07101 seconds |
9
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:2:2] | passed | 0.01402 seconds |
10
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:1] | passed | 0.01324 seconds |
11
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:2] | passed | 0.01055 seconds |
12
+ ./spec/models/lab_tech/experiment_spec.rb[1:1:2:4:1] | passed | 0.00826 seconds |
13
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:1] | passed | 0.00044 seconds |
14
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:2] | passed | 0.00034 seconds |
15
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:3] | passed | 0.00033 seconds |
16
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:4] | passed | 0.00043 seconds |
17
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:5] | passed | 0.00039 seconds |
18
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:6] | passed | 0.00033 seconds |
19
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:7] | passed | 0.00032 seconds |
20
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:8] | passed | 0.00034 seconds |
21
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:9] | passed | 0.00038 seconds |
22
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:1:10] | passed | 0.00038 seconds |
23
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:2:1] | passed | 0.00039 seconds |
24
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:2:2] | passed | 0.00034 seconds |
25
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:2:3] | passed | 0.00033 seconds |
26
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:2:4] | passed | 0.00031 seconds |
27
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:3:1] | passed | 0.00034 seconds |
28
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:3:2] | passed | 0.00032 seconds |
29
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:3:3] | passed | 0.00031 seconds |
30
+ ./spec/models/lab_tech/percentile_spec.rb[1:1:3:4] | passed | 0.0003 seconds |
31
+ ./spec/models/lab_tech/percentile_spec.rb[1:2:1] | passed | 0.00044 seconds |
32
+ ./spec/models/lab_tech/percentile_spec.rb[1:2:2] | passed | 0.00043 seconds |
33
+ ./spec/models/lab_tech/percentile_spec.rb[1:2:3] | passed | 0.00035 seconds |
34
+ ./spec/models/lab_tech/percentile_spec.rb[1:2:4] | passed | 0.00036 seconds |
35
+ ./spec/models/lab_tech/percentile_spec.rb[1:2:5] | passed | 0.00037 seconds |
36
+ ./spec/models/lab_tech/percentile_spec.rb[1:3:1] | passed | 0.00047 seconds |
37
+ ./spec/models/lab_tech/percentile_spec.rb[1:3:2] | passed | 0.00045 seconds |
38
+ ./spec/models/lab_tech/percentile_spec.rb[1:3:3] | passed | 0.00044 seconds |
39
+ ./spec/models/lab_tech/percentile_spec.rb[1:3:4] | passed | 0.00043 seconds |
40
+ ./spec/models/lab_tech/percentile_spec.rb[1:3:5] | passed | 0.00045 seconds |
41
+ ./spec/models/lab_tech/result_spec.rb[1:1:1] | passed | 0.01435 seconds |
42
+ ./spec/models/lab_tech/result_spec.rb[1:1:2:1:1] | passed | 0.00917 seconds |
43
+ ./spec/models/lab_tech/result_spec.rb[1:1:2:2:1] | passed | 0.01049 seconds |
44
+ ./spec/models/lab_tech/result_spec.rb[1:1:2:3:1] | passed | 0.00984 seconds |
45
+ ./spec/models/lab_tech/result_spec.rb[1:1:2:3:2] | passed | 0.00965 seconds |
46
+ ./spec/models/lab_tech/result_spec.rb[1:1:2:4:1] | passed | 0.00998 seconds |
47
+ ./spec/models/lab_tech/result_spec.rb[1:1:3:1] | passed | 0.00759 seconds |
48
+ ./spec/models/lab_tech/result_spec.rb[1:1:4:1] | passed | 0.01068 seconds |
49
+ ./spec/models/lab_tech/result_spec.rb[1:1:5:1] | passed | 0.01141 seconds |
50
+ ./spec/models/lab_tech/result_spec.rb[1:1:6] | passed | 0.01769 seconds |
51
+ ./spec/models/lab_tech/speedup_spec.rb[1:1] | passed | 0.00127 seconds |
52
+ ./spec/models/lab_tech/speedup_spec.rb[1:2] | passed | 0.0004 seconds |
53
+ ./spec/models/lab_tech/speedup_spec.rb[1:3] | passed | 0.00037 seconds |
54
+ ./spec/models/lab_tech/speedup_spec.rb[1:4] | passed | 0.00036 seconds |
55
+ ./spec/models/lab_tech/speedup_spec.rb[1:5] | passed | 0.00036 seconds |
56
+ ./spec/models/lab_tech/speedup_spec.rb[1:6] | passed | 0.00036 seconds |
57
+ ./spec/models/lab_tech/speedup_spec.rb[1:7] | passed | 0.00035 seconds |
58
+ ./spec/models/lab_tech/speedup_spec.rb[1:8] | passed | 0.00036 seconds |
59
+ ./spec/models/lab_tech/speedup_spec.rb[1:9] | passed | 0.00055 seconds |
60
+ ./spec/models/lab_tech/speedup_spec.rb[1:10] | passed | 0.00045 seconds |
61
+ ./spec/models/lab_tech/speedup_spec.rb[1:11] | passed | 0.00037 seconds |
62
+ ./spec/models/lab_tech/speedup_spec.rb[1:12] | passed | 0.00047 seconds |
63
+ ./spec/models/lab_tech/speedup_spec.rb[1:13] | passed | 0.00048 seconds |
64
+ ./spec/models/lab_tech/summary_spec.rb[1:1:1] | passed | 0.0045 seconds |
65
+ ./spec/models/lab_tech/summary_spec.rb[1:2:1] | passed | 0.03011 seconds |
66
+ ./spec/models/lab_tech/summary_spec.rb[1:3:1] | passed | 0.01405 seconds |
67
+ ./spec/models/lab_tech/summary_spec.rb[1:4:1] | passed | 0.01349 seconds |
68
+ ./spec/models/lab_tech/summary_spec.rb[1:5:1] | passed | 0.01768 seconds |
69
+ ./spec/models/lab_tech/summary_spec.rb[1:5:2] | passed | 0.01745 seconds |
70
+ ./spec/models/lab_tech/summary_spec.rb[1:6:1:1] | passed | 0.0281 seconds |
71
+ ./spec/models/lab_tech/summary_spec.rb[1:6:1:2] | passed | 0.02124 seconds |
72
+ ./spec/models/lab_tech/summary_spec.rb[1:6:2:1] | passed | 0.02179 seconds |
73
+ ./spec/models/lab_tech/summary_spec.rb[1:6:3:1] | passed | 0.02039 seconds |
74
+ ./spec/models/lab_tech/summary_spec.rb[1:6:4:1] | passed | 0.07298 seconds |
75
+ ./spec/models/lab_tech/summary_spec.rb[1:6:4:2] | passed | 0.05676 seconds |
76
+ ./spec/models/lab_tech/summary_spec.rb[1:6:4:3] | passed | 0.05425 seconds |
77
+ ./spec/models/lab_tech/summary_spec.rb[1:6:5:1] | passed | 0.16329 seconds |
78
+ ./spec/models/lab_tech/summary_spec.rb[1:6:6:1] | passed | 0.16302 seconds |
79
+ ./spec/models/lab_tech_spec.rb[1:1:1] | passed | 0.00093 seconds |
@@ -0,0 +1,32 @@
1
+ require 'rails_helper'
2
+
3
+ RSpec.describe LabTech::DefaultCleaner, type: :model do
4
+ subject(:cleaner) { LabTech::DefaultCleaner }
5
+
6
+ def clean(value)
7
+ cleaner.call(value)
8
+ end
9
+
10
+ it "returns an integer as itself" do
11
+ expect( clean(42) ).to eq( 42 )
12
+ end
13
+
14
+ it "returns an AR instance as a pair of [ class_name, id ]" do
15
+ exp = LabTech::Experiment.create(name: "whatever")
16
+ expect( clean(exp) ).to eq( [ "LabTech::Experiment", exp.id ] )
17
+ end
18
+
19
+ it "returns an array of integers as itself" do
20
+ expect( clean( [1,2,3] ) ).to eq( [1,2,3] )
21
+ end
22
+
23
+ it "returns an array of AR instances as a hash containing a list of IDs keyed by class name" do
24
+ e1, e2 = 2.times.map {|i| LabTech::Experiment.create(name: "Experiment #{i}") }
25
+ expect( clean( [e1, e2] ) ).to eq(
26
+ {
27
+ "LabTech::Experiment" => [ e1.id, e2.id ],
28
+ }
29
+ )
30
+ end
31
+ end
32
+
@@ -0,0 +1,110 @@
1
+ require 'rails_helper'
2
+ require SPEC_ROOT.join('support/misc_helpers.rb')
3
+
4
+ RSpec.describe LabTech::Experiment do
5
+ around do |example|
6
+ LabTech.publish_results_in_test_mode do
7
+ example.run
8
+ end
9
+ end
10
+
11
+ def wtf
12
+ puts "", "#" * 100
13
+ puts "\nExperiments" ; tp LabTech::Experiment.all
14
+ puts "\nResults" ; tp LabTech::Result.all
15
+ puts "\nObservations" ; tp LabTech::Observation.all
16
+ puts "", "#" * 100
17
+ end
18
+
19
+ describe ".science" do
20
+ let!(:experiment) { described_class.create(name: "wibble", percent_enabled: 0) }
21
+
22
+ context "by default" do
23
+ it "runs the .use block (the 'control') but not the .try block (the 'candidate')" do
24
+ control, candidate = false, false
25
+ LabTech.science "wibble" do |e|
26
+ e.use { control = true }
27
+ e.try { candidate = true }
28
+ end
29
+
30
+ expect( control ).to be true
31
+ expect( candidate ).to be false
32
+ end
33
+ end
34
+
35
+ context "when the experiment is #enabled?" do
36
+ before do
37
+ experiment.update_attribute(:percent_enabled, 100)
38
+ end
39
+
40
+ it "runs the .try block (the 'candidate') when that experiment is #enabled?" do
41
+ control, candidate = false, false
42
+ LabTech.science "wibble" do |e|
43
+ e.use { control = true }
44
+ e.try { candidate = true }
45
+ end
46
+
47
+ expect( control ).to be true
48
+ expect( candidate ).to be true
49
+ end
50
+
51
+ it "records the results when the experiment is run" do
52
+ expect( LabTech::Result ).to receive( :record_a_science ).with( experiment, instance_of(Scientist::Result) )
53
+
54
+ LabTech.science "wibble" do |e|
55
+ e.use { :wibble }
56
+ e.try { :wobble }
57
+ end
58
+ end
59
+
60
+ describe "value-cleaning behavior" do
61
+ let(:result) { experiment.results.first }
62
+
63
+ specify "if a #clean block IS provided, it is used" do
64
+ LabTech.science "wibble" do |e|
65
+ e.use { :control }
66
+ e.try { :candidate }
67
+ e.clean { |value| value.to_s.upcase }
68
+ end
69
+
70
+ result = experiment.results.first
71
+ expect( result ).to be_kind_of( LabTech::Result )
72
+
73
+ expect( result.control .value ).to eq( "CONTROL" )
74
+ expect( result.candidates.first .value ).to eq( "CANDIDATE" )
75
+ end
76
+
77
+ specify "if a #clean block IS NOT provided, DefaultCleaner is used" do
78
+ default = LabTech::DefaultCleaner
79
+ expect( default ).to receive( :call ).with( :control ).and_return( "Yes indeedily!" )
80
+ expect( default ).to receive( :call ).with( :candidate ).and_return( "You suck-diddly-uck, Flanders!" )
81
+
82
+ LabTech.science "wibble" do |e|
83
+ e.use { :control }
84
+ e.try { :candidate }
85
+ end
86
+
87
+ result = experiment.results.first
88
+ expect( result.control .value ).to eq( "Yes indeedily!" )
89
+ expect( result.candidates.first .value ).to eq( "You suck-diddly-uck, Flanders!" )
90
+ end
91
+ end
92
+
93
+ describe "result counts" do
94
+ specify "when results are equivalent" do
95
+ LabTech.science "wibble" do |e|
96
+ e.use { :wibble }
97
+ e.try { :wibble }
98
+ end
99
+
100
+ experiment.reload
101
+ aggregate_failures do
102
+ expect( experiment.equivalent_count ).to eq( 1 )
103
+ expect( experiment.timed_out_count ).to eq( 0 )
104
+ expect( experiment.other_error_count ).to eq( 0 )
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,85 @@
1
+ require 'rails_helper'
2
+
3
+ RSpec.describe LabTech::Percentile do
4
+ # Just keep walking, Mr. Phippen, and nobody gets hurt. ;P
5
+ def self.expect_percentile( percentile, expected_value )
6
+ specify "percentile(#{percentile}) should be #{expected_value}" do
7
+ expect( LabTech::Percentile.call(percentile, array) ).to eq( expected_value )
8
+ end
9
+ end
10
+
11
+ def self.expect_percentiles(percentiles_to_values = {})
12
+ percentiles_to_values.each do |percentile, value|
13
+ expect_percentile percentile, value
14
+ end
15
+ end
16
+
17
+ describe "examples I swiped from Wikipedia" do
18
+ # Specifically: https://en.wikipedia.org/wiki/Percentile#The_nearest-rank_method
19
+ context "for a 5-item array" do
20
+ subject(:array) { [ 15, 20, 35, 40, 50 ] }
21
+
22
+ expect_percentiles({
23
+ 0 => 15,
24
+ 20 => 15,
25
+ 21 => 20,
26
+ 40 => 20,
27
+ 41 => 35,
28
+ 60 => 35,
29
+ 61 => 40,
30
+ 80 => 40,
31
+ 81 => 50,
32
+ 100 => 50,
33
+ })
34
+
35
+ end
36
+
37
+ context "for a 10-item array" do
38
+ subject(:array) { [ 3, 6, 7, 8, 8, 10, 13, 15, 16, 20 ] }
39
+
40
+ expect_percentiles({
41
+ 25 => 7,
42
+ 50 => 8,
43
+ 75 => 15,
44
+ 100 => 20,
45
+ })
46
+ end
47
+
48
+ context "for an 11-item array" do
49
+ subject(:array) { [ 3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20 ] }
50
+
51
+ expect_percentiles({
52
+ 25 => 7,
53
+ 50 => 9,
54
+ 75 => 15,
55
+ 100 => 20,
56
+ })
57
+
58
+ end
59
+ end
60
+
61
+ context "for a 100-item array" do
62
+ subject(:array) { (1..100).to_a }
63
+
64
+ expect_percentiles({
65
+ 0 => 1,
66
+ 1 => 1,
67
+ 2 => 2,
68
+ 99 => 99,
69
+ 100 => 100,
70
+ })
71
+ end
72
+
73
+ context "for a 1000-item array" do
74
+ subject(:array) { (1..1000).to_a }
75
+
76
+ expect_percentiles({
77
+ 0 => 1,
78
+ 1 => 10,
79
+ 50 => 500,
80
+ 99 => 990,
81
+ 100 => 1000,
82
+ })
83
+
84
+ end
85
+ end
@@ -0,0 +1,198 @@
1
+ require 'rails_helper'
2
+ require SPEC_ROOT.join('support/misc_helpers.rb')
3
+
4
+ RSpec.describe LabTech::Result, type: :model do
5
+ let!(:experiment) { LabTech::Experiment.create(name: "wibble", percent_enabled: 100) }
6
+
7
+ around do |example|
8
+ LabTech.publish_results_in_test_mode do
9
+ example.run
10
+ end
11
+ end
12
+
13
+ def wibble_wobble!( fabricated_durations: {} )
14
+ LabTech.science "wibble" do |e|
15
+ e.use { :wibble }
16
+ e.try { :wobble }
17
+
18
+ # As of 1.3.0, Scientist allows you to provide fake timing data :)
19
+ e.fabricate_durations_for_testing_purposes( fabricated_durations )
20
+ end
21
+ end
22
+
23
+ describe ".record_a_science" do
24
+ let(:result) { LabTech::Result.last }
25
+ let(:control) { result.control }
26
+ let(:candidate) { result.candidates.first }
27
+
28
+ it "creates records for the result and both observations" do
29
+ expect { wibble_wobble! } \
30
+ .to change { LabTech::Result .count }.by( 1 )
31
+ .and change { LabTech::Observation .count }.by( 2 )
32
+
33
+ aggregate_failures do
34
+ expect( result.equivalent ).to be false
35
+ expect( result.raised_error ).to be false
36
+
37
+ expect( result.control.value ).to eq( :wibble )
38
+ expect( result.candidates.first.value ).to eq( :wobble )
39
+ end
40
+ end
41
+
42
+ describe "timing data" do
43
+ context "when one behavior takes zero time" do
44
+ let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.0 } }
45
+
46
+ specify "the saved records contain timing data (durations, delta, but no speedup)" do
47
+ wibble_wobble! fabricated_durations: fabricated_durations
48
+
49
+ expect( control.duration ).to eq( 0.5 )
50
+ expect( candidate.duration ).to eq( 0.0 )
51
+ expect( result.control_duration ).to eq( 0.5 )
52
+ expect( result.candidate_duration ).to eq( 0.0 )
53
+ expect( result.time_delta ).to eq( 0.5 )
54
+ expect( result.speedup_factor ).to be nil
55
+ end
56
+ end
57
+
58
+ context "when both behaviors take zero time" do
59
+ let(:fabricated_durations) { { "control" => 0.0, "candidate" => 0.0 } }
60
+
61
+ specify "the saved records contain timing data (durations, delta, but no speedup)" do
62
+ wibble_wobble! fabricated_durations: { "control" => 0.0, "candidate" => 0.0 }
63
+
64
+ expect( control.duration ).to eq( 0.0 )
65
+ expect( candidate.duration ).to eq( 0.0 )
66
+ expect( result.control_duration ).to eq( 0.0 )
67
+ expect( result.candidate_duration ).to eq( 0.0 )
68
+ expect( result.time_delta ).to eq( 0.0 )
69
+ expect( result.speedup_factor ).to be nil
70
+ end
71
+ end
72
+
73
+ context "when both behaviors take exactly the same time" do
74
+ let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.5 } }
75
+
76
+ specify "the saved records contain timing data (durations, delta, speedup)" do
77
+ wibble_wobble! fabricated_durations: fabricated_durations
78
+
79
+ expect( control.duration ).to eq( 0.5 )
80
+ expect( candidate.duration ).to eq( 0.5 )
81
+ expect( result.control_duration ).to eq( 0.5 )
82
+ expect( result.candidate_duration ).to eq( 0.5 )
83
+ expect( result.time_delta ).to eq( 0.0 )
84
+ expect( result.speedup_factor ).to eq( 0.0 )
85
+ end
86
+
87
+ specify "the result has a Speedup object" do
88
+ wibble_wobble! fabricated_durations: fabricated_durations
89
+ speedup = result.speedup
90
+
91
+ expect( speedup ).to be_kind_of( LabTech::Speedup )
92
+
93
+ expect( speedup.time ).to eq( 0.0 )
94
+ expect( speedup.factor ).to eq( 0.0 )
95
+ end
96
+ end
97
+
98
+ context "when one behavior takes twice as long as the other" do
99
+ let(:fabricated_durations) { { "control" => 0.5, "candidate" => 1.0 } }
100
+
101
+ specify "the saved records contain timing data (durations, delta, speedup)" do
102
+ wibble_wobble! fabricated_durations: fabricated_durations
103
+
104
+ expect( control.duration ).to eq( 0.5 )
105
+ expect( candidate.duration ).to eq( 1.0 )
106
+ expect( result.time_delta ).to eq( -0.5 )
107
+ expect( result.speedup_factor ).to eq( -2.0 )
108
+ end
109
+ end
110
+ end
111
+
112
+ context "when a comparator is provided" do
113
+ before do
114
+ LabTech.science "wibble" do |e|
115
+ e.use { :wibble }
116
+ e.try { :WIBBLE }
117
+ e.compare { |control, candidate| control.to_s.upcase == candidate.to_s.upcase }
118
+ end
119
+ end
120
+
121
+ specify "we use it to check equivalency" do
122
+ expect( result ).to be_equivalent
123
+ end
124
+ end
125
+
126
+ context "when the candidate raises an exception" do
127
+ before do
128
+ LabTech.science "wibble" do |e|
129
+ e.use { :wibble }
130
+ e.try { raise "nope" }
131
+ end
132
+ end
133
+
134
+ specify "we don't asplode" do
135
+ aggregate_failures do
136
+ expect( result .raised_error? ).to be true
137
+ expect( control .raised_error? ).to be false
138
+ expect( candidate.raised_error? ).to be true
139
+
140
+ expect( result .timed_out? ).to be false
141
+ expect( control .timed_out? ).to be false
142
+ expect( candidate.timed_out? ).to be false
143
+
144
+ expect( candidate.exception_class ).to eq( "RuntimeError" )
145
+ expect( candidate.exception_message ).to eq( "nope" )
146
+ expect( candidate.exception_backtrace ).to be_present
147
+ end
148
+ end
149
+ end
150
+
151
+ context "when the exception raised is a Timeout::Error" do
152
+ before do
153
+ LabTech.science "wibble" do |e|
154
+ e.use { :wibble }
155
+ e.try { raise Timeout::Error, "nope" }
156
+ end
157
+ end
158
+
159
+ specify "we mark the result as :timed_out AND :raised_error" do
160
+ aggregate_failures do
161
+ expect( result .raised_error? ).to be true
162
+ expect( control .raised_error? ).to be false
163
+ expect( candidate.raised_error? ).to be true
164
+
165
+ expect( result .timed_out? ).to be true
166
+ expect( control .timed_out? ).to be false
167
+ expect( candidate.timed_out? ).to be true
168
+
169
+ expect( candidate.exception_class ).to eq( "Timeout::Error" )
170
+ expect( candidate.exception_message ).to eq( "nope" )
171
+ expect( candidate.exception_backtrace ).to be_present
172
+ end
173
+
174
+ expect( LabTech::Result.timed_out ).to include( result )
175
+ end
176
+ end
177
+
178
+ specify "Results that time out are *not* also counted as mismatches" do
179
+ LabTech.science "wibble" do |e|
180
+ e.use { :wibble }
181
+ e.try { raise Timeout::Error, "nope" }
182
+ end
183
+
184
+ aggregate_failures do
185
+ expect( result.equivalent? ).to be false
186
+ expect( result.timed_out? ).to be true
187
+ expect( result.raised_error? ).to be true
188
+
189
+ expect( described_class.correct ).to_not include( result )
190
+ expect( described_class.mismatched ).to_not include( result )
191
+ expect( described_class.timed_out ).to include( result )
192
+ expect( described_class.other_error ).to_not include( result )
193
+ end
194
+
195
+ expect( LabTech::Result.timed_out ).to include( result )
196
+ end
197
+ end
198
+ end