lab_tech 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/MIT-LICENSE +20 -0
- data/README.md +323 -0
- data/Rakefile +30 -0
- data/app/models/lab_tech/application_record.rb +5 -0
- data/app/models/lab_tech/default_cleaner.rb +87 -0
- data/app/models/lab_tech/experiment.rb +190 -0
- data/app/models/lab_tech/observation.rb +40 -0
- data/app/models/lab_tech/percentile.rb +41 -0
- data/app/models/lab_tech/result.rb +130 -0
- data/app/models/lab_tech/speedup.rb +65 -0
- data/app/models/lab_tech/summary.rb +183 -0
- data/config/routes.rb +2 -0
- data/db/migrate/20190815192130_create_experiment_tables.rb +50 -0
- data/lib/lab_tech.rb +176 -0
- data/lib/lab_tech/engine.rb +6 -0
- data/lib/lab_tech/version.rb +3 -0
- data/lib/tasks/lab_tech_tasks.rake +4 -0
- data/spec/dummy/Rakefile +6 -0
- data/spec/dummy/app/assets/config/manifest.js +1 -0
- data/spec/dummy/app/assets/javascripts/application.js +14 -0
- data/spec/dummy/app/assets/stylesheets/application.css +15 -0
- data/spec/dummy/app/controllers/application_controller.rb +2 -0
- data/spec/dummy/app/jobs/application_job.rb +2 -0
- data/spec/dummy/app/models/application_record.rb +3 -0
- data/spec/dummy/bin/bundle +3 -0
- data/spec/dummy/bin/rails +4 -0
- data/spec/dummy/bin/rake +4 -0
- data/spec/dummy/bin/setup +33 -0
- data/spec/dummy/bin/update +28 -0
- data/spec/dummy/config.ru +5 -0
- data/spec/dummy/config/application.rb +35 -0
- data/spec/dummy/config/boot.rb +5 -0
- data/spec/dummy/config/database.yml +25 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +46 -0
- data/spec/dummy/config/environments/production.rb +71 -0
- data/spec/dummy/config/environments/test.rb +36 -0
- data/spec/dummy/config/initializers/application_controller_renderer.rb +8 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/cors.rb +16 -0
- data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
- data/spec/dummy/config/initializers/inflections.rb +16 -0
- data/spec/dummy/config/initializers/mime_types.rb +4 -0
- data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/spec/dummy/config/locales/en.yml +33 -0
- data/spec/dummy/config/puma.rb +34 -0
- data/spec/dummy/config/routes.rb +3 -0
- data/spec/dummy/config/spring.rb +6 -0
- data/spec/dummy/db/schema.rb +52 -0
- data/spec/dummy/db/test.sqlite3 +0 -0
- data/spec/dummy/log/development.log +0 -0
- data/spec/dummy/log/test.log +1519 -0
- data/spec/examples.txt +79 -0
- data/spec/models/lab_tech/default_cleaner_spec.rb +32 -0
- data/spec/models/lab_tech/experiment_spec.rb +110 -0
- data/spec/models/lab_tech/percentile_spec.rb +85 -0
- data/spec/models/lab_tech/result_spec.rb +198 -0
- data/spec/models/lab_tech/speedup_spec.rb +133 -0
- data/spec/models/lab_tech/summary_spec.rb +325 -0
- data/spec/models/lab_tech_spec.rb +23 -0
- data/spec/rails_helper.rb +62 -0
- data/spec/spec_helper.rb +98 -0
- data/spec/support/misc_helpers.rb +7 -0
- metadata +238 -0
data/spec/examples.txt
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
example_id | status | run_time |
|
2
|
+
---------------------------------------------------- | ------ | --------------- |
|
3
|
+
./spec/models/lab_tech/default_cleaner_spec.rb[1:1] | passed | 0.00112 seconds |
|
4
|
+
./spec/models/lab_tech/default_cleaner_spec.rb[1:2] | passed | 0.0078 seconds |
|
5
|
+
./spec/models/lab_tech/default_cleaner_spec.rb[1:3] | passed | 0.00039 seconds |
|
6
|
+
./spec/models/lab_tech/default_cleaner_spec.rb[1:4] | passed | 0.00198 seconds |
|
7
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:1:1] | passed | 0.00335 seconds |
|
8
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:2:1] | passed | 0.07101 seconds |
|
9
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:2:2] | passed | 0.01402 seconds |
|
10
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:1] | passed | 0.01324 seconds |
|
11
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:2:3:2] | passed | 0.01055 seconds |
|
12
|
+
./spec/models/lab_tech/experiment_spec.rb[1:1:2:4:1] | passed | 0.00826 seconds |
|
13
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:1] | passed | 0.00044 seconds |
|
14
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:2] | passed | 0.00034 seconds |
|
15
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:3] | passed | 0.00033 seconds |
|
16
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:4] | passed | 0.00043 seconds |
|
17
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:5] | passed | 0.00039 seconds |
|
18
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:6] | passed | 0.00033 seconds |
|
19
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:7] | passed | 0.00032 seconds |
|
20
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:8] | passed | 0.00034 seconds |
|
21
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:9] | passed | 0.00038 seconds |
|
22
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:1:10] | passed | 0.00038 seconds |
|
23
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:2:1] | passed | 0.00039 seconds |
|
24
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:2:2] | passed | 0.00034 seconds |
|
25
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:2:3] | passed | 0.00033 seconds |
|
26
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:2:4] | passed | 0.00031 seconds |
|
27
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:3:1] | passed | 0.00034 seconds |
|
28
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:3:2] | passed | 0.00032 seconds |
|
29
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:3:3] | passed | 0.00031 seconds |
|
30
|
+
./spec/models/lab_tech/percentile_spec.rb[1:1:3:4] | passed | 0.0003 seconds |
|
31
|
+
./spec/models/lab_tech/percentile_spec.rb[1:2:1] | passed | 0.00044 seconds |
|
32
|
+
./spec/models/lab_tech/percentile_spec.rb[1:2:2] | passed | 0.00043 seconds |
|
33
|
+
./spec/models/lab_tech/percentile_spec.rb[1:2:3] | passed | 0.00035 seconds |
|
34
|
+
./spec/models/lab_tech/percentile_spec.rb[1:2:4] | passed | 0.00036 seconds |
|
35
|
+
./spec/models/lab_tech/percentile_spec.rb[1:2:5] | passed | 0.00037 seconds |
|
36
|
+
./spec/models/lab_tech/percentile_spec.rb[1:3:1] | passed | 0.00047 seconds |
|
37
|
+
./spec/models/lab_tech/percentile_spec.rb[1:3:2] | passed | 0.00045 seconds |
|
38
|
+
./spec/models/lab_tech/percentile_spec.rb[1:3:3] | passed | 0.00044 seconds |
|
39
|
+
./spec/models/lab_tech/percentile_spec.rb[1:3:4] | passed | 0.00043 seconds |
|
40
|
+
./spec/models/lab_tech/percentile_spec.rb[1:3:5] | passed | 0.00045 seconds |
|
41
|
+
./spec/models/lab_tech/result_spec.rb[1:1:1] | passed | 0.01435 seconds |
|
42
|
+
./spec/models/lab_tech/result_spec.rb[1:1:2:1:1] | passed | 0.00917 seconds |
|
43
|
+
./spec/models/lab_tech/result_spec.rb[1:1:2:2:1] | passed | 0.01049 seconds |
|
44
|
+
./spec/models/lab_tech/result_spec.rb[1:1:2:3:1] | passed | 0.00984 seconds |
|
45
|
+
./spec/models/lab_tech/result_spec.rb[1:1:2:3:2] | passed | 0.00965 seconds |
|
46
|
+
./spec/models/lab_tech/result_spec.rb[1:1:2:4:1] | passed | 0.00998 seconds |
|
47
|
+
./spec/models/lab_tech/result_spec.rb[1:1:3:1] | passed | 0.00759 seconds |
|
48
|
+
./spec/models/lab_tech/result_spec.rb[1:1:4:1] | passed | 0.01068 seconds |
|
49
|
+
./spec/models/lab_tech/result_spec.rb[1:1:5:1] | passed | 0.01141 seconds |
|
50
|
+
./spec/models/lab_tech/result_spec.rb[1:1:6] | passed | 0.01769 seconds |
|
51
|
+
./spec/models/lab_tech/speedup_spec.rb[1:1] | passed | 0.00127 seconds |
|
52
|
+
./spec/models/lab_tech/speedup_spec.rb[1:2] | passed | 0.0004 seconds |
|
53
|
+
./spec/models/lab_tech/speedup_spec.rb[1:3] | passed | 0.00037 seconds |
|
54
|
+
./spec/models/lab_tech/speedup_spec.rb[1:4] | passed | 0.00036 seconds |
|
55
|
+
./spec/models/lab_tech/speedup_spec.rb[1:5] | passed | 0.00036 seconds |
|
56
|
+
./spec/models/lab_tech/speedup_spec.rb[1:6] | passed | 0.00036 seconds |
|
57
|
+
./spec/models/lab_tech/speedup_spec.rb[1:7] | passed | 0.00035 seconds |
|
58
|
+
./spec/models/lab_tech/speedup_spec.rb[1:8] | passed | 0.00036 seconds |
|
59
|
+
./spec/models/lab_tech/speedup_spec.rb[1:9] | passed | 0.00055 seconds |
|
60
|
+
./spec/models/lab_tech/speedup_spec.rb[1:10] | passed | 0.00045 seconds |
|
61
|
+
./spec/models/lab_tech/speedup_spec.rb[1:11] | passed | 0.00037 seconds |
|
62
|
+
./spec/models/lab_tech/speedup_spec.rb[1:12] | passed | 0.00047 seconds |
|
63
|
+
./spec/models/lab_tech/speedup_spec.rb[1:13] | passed | 0.00048 seconds |
|
64
|
+
./spec/models/lab_tech/summary_spec.rb[1:1:1] | passed | 0.0045 seconds |
|
65
|
+
./spec/models/lab_tech/summary_spec.rb[1:2:1] | passed | 0.03011 seconds |
|
66
|
+
./spec/models/lab_tech/summary_spec.rb[1:3:1] | passed | 0.01405 seconds |
|
67
|
+
./spec/models/lab_tech/summary_spec.rb[1:4:1] | passed | 0.01349 seconds |
|
68
|
+
./spec/models/lab_tech/summary_spec.rb[1:5:1] | passed | 0.01768 seconds |
|
69
|
+
./spec/models/lab_tech/summary_spec.rb[1:5:2] | passed | 0.01745 seconds |
|
70
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:1:1] | passed | 0.0281 seconds |
|
71
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:1:2] | passed | 0.02124 seconds |
|
72
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:2:1] | passed | 0.02179 seconds |
|
73
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:3:1] | passed | 0.02039 seconds |
|
74
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:4:1] | passed | 0.07298 seconds |
|
75
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:4:2] | passed | 0.05676 seconds |
|
76
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:4:3] | passed | 0.05425 seconds |
|
77
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:5:1] | passed | 0.16329 seconds |
|
78
|
+
./spec/models/lab_tech/summary_spec.rb[1:6:6:1] | passed | 0.16302 seconds |
|
79
|
+
./spec/models/lab_tech_spec.rb[1:1:1] | passed | 0.00093 seconds |
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
|
3
|
+
RSpec.describe LabTech::DefaultCleaner, type: :model do
|
4
|
+
subject(:cleaner) { LabTech::DefaultCleaner }
|
5
|
+
|
6
|
+
def clean(value)
|
7
|
+
cleaner.call(value)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "returns an integer as itself" do
|
11
|
+
expect( clean(42) ).to eq( 42 )
|
12
|
+
end
|
13
|
+
|
14
|
+
it "returns an AR instance as a pair of [ class_name, id ]" do
|
15
|
+
exp = LabTech::Experiment.create(name: "whatever")
|
16
|
+
expect( clean(exp) ).to eq( [ "LabTech::Experiment", exp.id ] )
|
17
|
+
end
|
18
|
+
|
19
|
+
it "returns an array of integers as itself" do
|
20
|
+
expect( clean( [1,2,3] ) ).to eq( [1,2,3] )
|
21
|
+
end
|
22
|
+
|
23
|
+
it "returns an array of AR instances as a hash containing a list of IDs keyed by class name" do
|
24
|
+
e1, e2 = 2.times.map {|i| LabTech::Experiment.create(name: "Experiment #{i}") }
|
25
|
+
expect( clean( [e1, e2] ) ).to eq(
|
26
|
+
{
|
27
|
+
"LabTech::Experiment" => [ e1.id, e2.id ],
|
28
|
+
}
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
require SPEC_ROOT.join('support/misc_helpers.rb')
|
3
|
+
|
4
|
+
RSpec.describe LabTech::Experiment do
|
5
|
+
around do |example|
|
6
|
+
LabTech.publish_results_in_test_mode do
|
7
|
+
example.run
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def wtf
|
12
|
+
puts "", "#" * 100
|
13
|
+
puts "\nExperiments" ; tp LabTech::Experiment.all
|
14
|
+
puts "\nResults" ; tp LabTech::Result.all
|
15
|
+
puts "\nObservations" ; tp LabTech::Observation.all
|
16
|
+
puts "", "#" * 100
|
17
|
+
end
|
18
|
+
|
19
|
+
describe ".science" do
|
20
|
+
let!(:experiment) { described_class.create(name: "wibble", percent_enabled: 0) }
|
21
|
+
|
22
|
+
context "by default" do
|
23
|
+
it "runs the .use block (the 'control') but not the .try block (the 'candidate')" do
|
24
|
+
control, candidate = false, false
|
25
|
+
LabTech.science "wibble" do |e|
|
26
|
+
e.use { control = true }
|
27
|
+
e.try { candidate = true }
|
28
|
+
end
|
29
|
+
|
30
|
+
expect( control ).to be true
|
31
|
+
expect( candidate ).to be false
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
context "when the experiment is #enabled?" do
|
36
|
+
before do
|
37
|
+
experiment.update_attribute(:percent_enabled, 100)
|
38
|
+
end
|
39
|
+
|
40
|
+
it "runs the .try block (the 'candidate') when that experiment is #enabled?" do
|
41
|
+
control, candidate = false, false
|
42
|
+
LabTech.science "wibble" do |e|
|
43
|
+
e.use { control = true }
|
44
|
+
e.try { candidate = true }
|
45
|
+
end
|
46
|
+
|
47
|
+
expect( control ).to be true
|
48
|
+
expect( candidate ).to be true
|
49
|
+
end
|
50
|
+
|
51
|
+
it "records the results when the experiment is run" do
|
52
|
+
expect( LabTech::Result ).to receive( :record_a_science ).with( experiment, instance_of(Scientist::Result) )
|
53
|
+
|
54
|
+
LabTech.science "wibble" do |e|
|
55
|
+
e.use { :wibble }
|
56
|
+
e.try { :wobble }
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
describe "value-cleaning behavior" do
|
61
|
+
let(:result) { experiment.results.first }
|
62
|
+
|
63
|
+
specify "if a #clean block IS provided, it is used" do
|
64
|
+
LabTech.science "wibble" do |e|
|
65
|
+
e.use { :control }
|
66
|
+
e.try { :candidate }
|
67
|
+
e.clean { |value| value.to_s.upcase }
|
68
|
+
end
|
69
|
+
|
70
|
+
result = experiment.results.first
|
71
|
+
expect( result ).to be_kind_of( LabTech::Result )
|
72
|
+
|
73
|
+
expect( result.control .value ).to eq( "CONTROL" )
|
74
|
+
expect( result.candidates.first .value ).to eq( "CANDIDATE" )
|
75
|
+
end
|
76
|
+
|
77
|
+
specify "if a #clean block IS NOT provided, DefaultCleaner is used" do
|
78
|
+
default = LabTech::DefaultCleaner
|
79
|
+
expect( default ).to receive( :call ).with( :control ).and_return( "Yes indeedily!" )
|
80
|
+
expect( default ).to receive( :call ).with( :candidate ).and_return( "You suck-diddly-uck, Flanders!" )
|
81
|
+
|
82
|
+
LabTech.science "wibble" do |e|
|
83
|
+
e.use { :control }
|
84
|
+
e.try { :candidate }
|
85
|
+
end
|
86
|
+
|
87
|
+
result = experiment.results.first
|
88
|
+
expect( result.control .value ).to eq( "Yes indeedily!" )
|
89
|
+
expect( result.candidates.first .value ).to eq( "You suck-diddly-uck, Flanders!" )
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
describe "result counts" do
|
94
|
+
specify "when results are equivalent" do
|
95
|
+
LabTech.science "wibble" do |e|
|
96
|
+
e.use { :wibble }
|
97
|
+
e.try { :wibble }
|
98
|
+
end
|
99
|
+
|
100
|
+
experiment.reload
|
101
|
+
aggregate_failures do
|
102
|
+
expect( experiment.equivalent_count ).to eq( 1 )
|
103
|
+
expect( experiment.timed_out_count ).to eq( 0 )
|
104
|
+
expect( experiment.other_error_count ).to eq( 0 )
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
|
3
|
+
RSpec.describe LabTech::Percentile do
|
4
|
+
# Just keep walking, Mr. Phippen, and nobody gets hurt. ;P
|
5
|
+
def self.expect_percentile( percentile, expected_value )
|
6
|
+
specify "percentile(#{percentile}) should be #{expected_value}" do
|
7
|
+
expect( LabTech::Percentile.call(percentile, array) ).to eq( expected_value )
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.expect_percentiles(percentiles_to_values = {})
|
12
|
+
percentiles_to_values.each do |percentile, value|
|
13
|
+
expect_percentile percentile, value
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe "examples I swiped from Wikipedia" do
|
18
|
+
# Specifically: https://en.wikipedia.org/wiki/Percentile#The_nearest-rank_method
|
19
|
+
context "for a 5-item array" do
|
20
|
+
subject(:array) { [ 15, 20, 35, 40, 50 ] }
|
21
|
+
|
22
|
+
expect_percentiles({
|
23
|
+
0 => 15,
|
24
|
+
20 => 15,
|
25
|
+
21 => 20,
|
26
|
+
40 => 20,
|
27
|
+
41 => 35,
|
28
|
+
60 => 35,
|
29
|
+
61 => 40,
|
30
|
+
80 => 40,
|
31
|
+
81 => 50,
|
32
|
+
100 => 50,
|
33
|
+
})
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
context "for a 10-item array" do
|
38
|
+
subject(:array) { [ 3, 6, 7, 8, 8, 10, 13, 15, 16, 20 ] }
|
39
|
+
|
40
|
+
expect_percentiles({
|
41
|
+
25 => 7,
|
42
|
+
50 => 8,
|
43
|
+
75 => 15,
|
44
|
+
100 => 20,
|
45
|
+
})
|
46
|
+
end
|
47
|
+
|
48
|
+
context "for an 11-item array" do
|
49
|
+
subject(:array) { [ 3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20 ] }
|
50
|
+
|
51
|
+
expect_percentiles({
|
52
|
+
25 => 7,
|
53
|
+
50 => 9,
|
54
|
+
75 => 15,
|
55
|
+
100 => 20,
|
56
|
+
})
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context "for a 100-item array" do
|
62
|
+
subject(:array) { (1..100).to_a }
|
63
|
+
|
64
|
+
expect_percentiles({
|
65
|
+
0 => 1,
|
66
|
+
1 => 1,
|
67
|
+
2 => 2,
|
68
|
+
99 => 99,
|
69
|
+
100 => 100,
|
70
|
+
})
|
71
|
+
end
|
72
|
+
|
73
|
+
context "for a 1000-item array" do
|
74
|
+
subject(:array) { (1..1000).to_a }
|
75
|
+
|
76
|
+
expect_percentiles({
|
77
|
+
0 => 1,
|
78
|
+
1 => 10,
|
79
|
+
50 => 500,
|
80
|
+
99 => 990,
|
81
|
+
100 => 1000,
|
82
|
+
})
|
83
|
+
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,198 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
require SPEC_ROOT.join('support/misc_helpers.rb')
|
3
|
+
|
4
|
+
RSpec.describe LabTech::Result, type: :model do
|
5
|
+
let!(:experiment) { LabTech::Experiment.create(name: "wibble", percent_enabled: 100) }
|
6
|
+
|
7
|
+
around do |example|
|
8
|
+
LabTech.publish_results_in_test_mode do
|
9
|
+
example.run
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def wibble_wobble!( fabricated_durations: {} )
|
14
|
+
LabTech.science "wibble" do |e|
|
15
|
+
e.use { :wibble }
|
16
|
+
e.try { :wobble }
|
17
|
+
|
18
|
+
# As of 1.3.0, Scientist allows you to provide fake timing data :)
|
19
|
+
e.fabricate_durations_for_testing_purposes( fabricated_durations )
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe ".record_a_science" do
|
24
|
+
let(:result) { LabTech::Result.last }
|
25
|
+
let(:control) { result.control }
|
26
|
+
let(:candidate) { result.candidates.first }
|
27
|
+
|
28
|
+
it "creates records for the result and both observations" do
|
29
|
+
expect { wibble_wobble! } \
|
30
|
+
.to change { LabTech::Result .count }.by( 1 )
|
31
|
+
.and change { LabTech::Observation .count }.by( 2 )
|
32
|
+
|
33
|
+
aggregate_failures do
|
34
|
+
expect( result.equivalent ).to be false
|
35
|
+
expect( result.raised_error ).to be false
|
36
|
+
|
37
|
+
expect( result.control.value ).to eq( :wibble )
|
38
|
+
expect( result.candidates.first.value ).to eq( :wobble )
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe "timing data" do
|
43
|
+
context "when one behavior takes zero time" do
|
44
|
+
let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.0 } }
|
45
|
+
|
46
|
+
specify "the saved records contain timing data (durations, delta, but no speedup)" do
|
47
|
+
wibble_wobble! fabricated_durations: fabricated_durations
|
48
|
+
|
49
|
+
expect( control.duration ).to eq( 0.5 )
|
50
|
+
expect( candidate.duration ).to eq( 0.0 )
|
51
|
+
expect( result.control_duration ).to eq( 0.5 )
|
52
|
+
expect( result.candidate_duration ).to eq( 0.0 )
|
53
|
+
expect( result.time_delta ).to eq( 0.5 )
|
54
|
+
expect( result.speedup_factor ).to be nil
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context "when both behaviors take zero time" do
|
59
|
+
let(:fabricated_durations) { { "control" => 0.0, "candidate" => 0.0 } }
|
60
|
+
|
61
|
+
specify "the saved records contain timing data (durations, delta, but no speedup)" do
|
62
|
+
wibble_wobble! fabricated_durations: { "control" => 0.0, "candidate" => 0.0 }
|
63
|
+
|
64
|
+
expect( control.duration ).to eq( 0.0 )
|
65
|
+
expect( candidate.duration ).to eq( 0.0 )
|
66
|
+
expect( result.control_duration ).to eq( 0.0 )
|
67
|
+
expect( result.candidate_duration ).to eq( 0.0 )
|
68
|
+
expect( result.time_delta ).to eq( 0.0 )
|
69
|
+
expect( result.speedup_factor ).to be nil
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
context "when both behaviors take exactly the same time" do
|
74
|
+
let(:fabricated_durations) { { "control" => 0.5, "candidate" => 0.5 } }
|
75
|
+
|
76
|
+
specify "the saved records contain timing data (durations, delta, speedup)" do
|
77
|
+
wibble_wobble! fabricated_durations: fabricated_durations
|
78
|
+
|
79
|
+
expect( control.duration ).to eq( 0.5 )
|
80
|
+
expect( candidate.duration ).to eq( 0.5 )
|
81
|
+
expect( result.control_duration ).to eq( 0.5 )
|
82
|
+
expect( result.candidate_duration ).to eq( 0.5 )
|
83
|
+
expect( result.time_delta ).to eq( 0.0 )
|
84
|
+
expect( result.speedup_factor ).to eq( 0.0 )
|
85
|
+
end
|
86
|
+
|
87
|
+
specify "the result has a Speedup object" do
|
88
|
+
wibble_wobble! fabricated_durations: fabricated_durations
|
89
|
+
speedup = result.speedup
|
90
|
+
|
91
|
+
expect( speedup ).to be_kind_of( LabTech::Speedup )
|
92
|
+
|
93
|
+
expect( speedup.time ).to eq( 0.0 )
|
94
|
+
expect( speedup.factor ).to eq( 0.0 )
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
context "when one behavior takes twice as long as the other" do
|
99
|
+
let(:fabricated_durations) { { "control" => 0.5, "candidate" => 1.0 } }
|
100
|
+
|
101
|
+
specify "the saved records contain timing data (durations, delta, speedup)" do
|
102
|
+
wibble_wobble! fabricated_durations: fabricated_durations
|
103
|
+
|
104
|
+
expect( control.duration ).to eq( 0.5 )
|
105
|
+
expect( candidate.duration ).to eq( 1.0 )
|
106
|
+
expect( result.time_delta ).to eq( -0.5 )
|
107
|
+
expect( result.speedup_factor ).to eq( -2.0 )
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
context "when a comparator is provided" do
|
113
|
+
before do
|
114
|
+
LabTech.science "wibble" do |e|
|
115
|
+
e.use { :wibble }
|
116
|
+
e.try { :WIBBLE }
|
117
|
+
e.compare { |control, candidate| control.to_s.upcase == candidate.to_s.upcase }
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
specify "we use it to check equivalency" do
|
122
|
+
expect( result ).to be_equivalent
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context "when the candidate raises an exception" do
|
127
|
+
before do
|
128
|
+
LabTech.science "wibble" do |e|
|
129
|
+
e.use { :wibble }
|
130
|
+
e.try { raise "nope" }
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
specify "we don't asplode" do
|
135
|
+
aggregate_failures do
|
136
|
+
expect( result .raised_error? ).to be true
|
137
|
+
expect( control .raised_error? ).to be false
|
138
|
+
expect( candidate.raised_error? ).to be true
|
139
|
+
|
140
|
+
expect( result .timed_out? ).to be false
|
141
|
+
expect( control .timed_out? ).to be false
|
142
|
+
expect( candidate.timed_out? ).to be false
|
143
|
+
|
144
|
+
expect( candidate.exception_class ).to eq( "RuntimeError" )
|
145
|
+
expect( candidate.exception_message ).to eq( "nope" )
|
146
|
+
expect( candidate.exception_backtrace ).to be_present
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
context "when the exception raised is a Timeout::Error" do
|
152
|
+
before do
|
153
|
+
LabTech.science "wibble" do |e|
|
154
|
+
e.use { :wibble }
|
155
|
+
e.try { raise Timeout::Error, "nope" }
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
specify "we mark the result as :timed_out AND :raised_error" do
|
160
|
+
aggregate_failures do
|
161
|
+
expect( result .raised_error? ).to be true
|
162
|
+
expect( control .raised_error? ).to be false
|
163
|
+
expect( candidate.raised_error? ).to be true
|
164
|
+
|
165
|
+
expect( result .timed_out? ).to be true
|
166
|
+
expect( control .timed_out? ).to be false
|
167
|
+
expect( candidate.timed_out? ).to be true
|
168
|
+
|
169
|
+
expect( candidate.exception_class ).to eq( "Timeout::Error" )
|
170
|
+
expect( candidate.exception_message ).to eq( "nope" )
|
171
|
+
expect( candidate.exception_backtrace ).to be_present
|
172
|
+
end
|
173
|
+
|
174
|
+
expect( LabTech::Result.timed_out ).to include( result )
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
specify "Results that time out are *not* also counted as mismatches" do
|
179
|
+
LabTech.science "wibble" do |e|
|
180
|
+
e.use { :wibble }
|
181
|
+
e.try { raise Timeout::Error, "nope" }
|
182
|
+
end
|
183
|
+
|
184
|
+
aggregate_failures do
|
185
|
+
expect( result.equivalent? ).to be false
|
186
|
+
expect( result.timed_out? ).to be true
|
187
|
+
expect( result.raised_error? ).to be true
|
188
|
+
|
189
|
+
expect( described_class.correct ).to_not include( result )
|
190
|
+
expect( described_class.mismatched ).to_not include( result )
|
191
|
+
expect( described_class.timed_out ).to include( result )
|
192
|
+
expect( described_class.other_error ).to_not include( result )
|
193
|
+
end
|
194
|
+
|
195
|
+
expect( LabTech::Result.timed_out ).to include( result )
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|