dat-science 1.2.0 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +9 -2
- data/dat-science.gemspec +3 -3
- data/lib/dat/science/experiment.rb +12 -1
- data/lib/dat/science/result.rb +10 -1
- data/test/dat_science_experiment_test.rb +23 -1
- data/test/dat_science_test.rb +1 -1
- metadata +10 -10
data/README.md
CHANGED
@@ -126,7 +126,7 @@ end
|
|
126
126
|
|
127
127
|
By default the results of an experiment are discarded. This isn't very useful.
|
128
128
|
`Experiment#publish` can be overridden to publish results via any
|
129
|
-
instrumentation
|
129
|
+
instrumentation mechanism, which makes it easy to graph durations or
|
130
130
|
matches/mismatches and store results. The only two events published by an
|
131
131
|
experiment are `:match` when the result of the control and candidate behaviors
|
132
132
|
are the same, and `:mismatch` when they aren't.
|
@@ -201,10 +201,17 @@ end
|
|
201
201
|
```
|
202
202
|
|
203
203
|
The results of the `control` and `candidate` blocks will be run through the
|
204
|
-
`cleaner
|
204
|
+
`cleaner`. You could get the same behavior by calling `count` in the blocks,
|
205
205
|
but the `cleaner` makes it easier to keep things in sync. The original
|
206
206
|
`control` result is still returned.
|
207
207
|
|
208
|
+
## What do I do with all these results?
|
209
|
+
|
210
|
+
Once you've started an experiment and published some results, you'll want to
|
211
|
+
analyze the mismatches from your experiment. Check out
|
212
|
+
[`dat-analysis`](https://github.com/github/dat-analysis) where you'll find an
|
213
|
+
analysis toolkit to help you understand your experiment results.
|
214
|
+
|
208
215
|
## Hacking on science
|
209
216
|
|
210
217
|
Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs
|
data/dat-science.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = "dat-science"
|
3
|
-
gem.version = "1.2.
|
3
|
+
gem.version = "1.2.1"
|
4
4
|
gem.authors = ["John Barnette", "Rick Bradley"]
|
5
5
|
gem.email = ["jbarnette@github.com"]
|
6
6
|
gem.description = "Gradually test, measure, and track refactored code."
|
@@ -12,6 +12,6 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep /^test/
|
13
13
|
gem.require_paths = ["lib"]
|
14
14
|
|
15
|
-
gem.add_development_dependency "minitest"
|
16
|
-
gem.add_development_dependency "mocha"
|
15
|
+
gem.add_development_dependency "minitest", "~> 5.0.8"
|
16
|
+
gem.add_development_dependency "mocha", "~> 0.14.0"
|
17
17
|
end
|
@@ -101,7 +101,7 @@ module Dat
|
|
101
101
|
:first => control_runs_first? ? :control : :candidate
|
102
102
|
}
|
103
103
|
|
104
|
-
kind = control
|
104
|
+
kind = evaluate control, candidate
|
105
105
|
publish_with_context kind, payload
|
106
106
|
|
107
107
|
raise control.exception if control.raised?
|
@@ -109,6 +109,17 @@ module Dat
|
|
109
109
|
control.value
|
110
110
|
end
|
111
111
|
|
112
|
+
# Public: Determine the outcome of an experiment run
|
113
|
+
#
|
114
|
+
# control - result from running control method
|
115
|
+
# candidate - result from running candidate method
|
116
|
+
#
|
117
|
+
# Returns :match if control and candidate produced the same
|
118
|
+
# result, :mismatch otherwise.
|
119
|
+
def evaluate(control, candidate)
|
120
|
+
control == candidate ? :match : :mismatch
|
121
|
+
end
|
122
|
+
|
112
123
|
protected
|
113
124
|
|
114
125
|
# Internal: Does this experiment have candidate behavior?
|
data/lib/dat/science/result.rb
CHANGED
@@ -37,11 +37,20 @@ module Dat
|
|
37
37
|
def payload
|
38
38
|
{
|
39
39
|
:duration => duration,
|
40
|
-
:exception =>
|
40
|
+
:exception => serialized_exception,
|
41
41
|
:value => experiment.clean(value)
|
42
42
|
}
|
43
43
|
end
|
44
44
|
|
45
|
+
def serialized_exception
|
46
|
+
return nil unless exception
|
47
|
+
{
|
48
|
+
:class => exception.class.name,
|
49
|
+
:message => exception.message,
|
50
|
+
:backtrace => exception.backtrace
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
45
54
|
def raised?
|
46
55
|
!!exception
|
47
56
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require "minitest/autorun"
|
2
2
|
require "dat/science/experiment"
|
3
3
|
|
4
|
-
class DatScienceExperimentTest < MiniTest::
|
4
|
+
class DatScienceExperimentTest < MiniTest::Test
|
5
5
|
class Experiment < Dat::Science::Experiment
|
6
6
|
def self.published
|
7
7
|
@published ||= []
|
@@ -257,4 +257,26 @@ class DatScienceExperimentTest < MiniTest::Unit::TestCase
|
|
257
257
|
refute_nil payload[:control][:exception]
|
258
258
|
refute_nil payload[:candidate][:exception]
|
259
259
|
end
|
260
|
+
|
261
|
+
def test_publishes_exception_data
|
262
|
+
e = Experiment.new "foo"
|
263
|
+
e.control { raise "foo" }
|
264
|
+
e.candidate { raise "bar" }
|
265
|
+
|
266
|
+
assert_raises RuntimeError do
|
267
|
+
e.run
|
268
|
+
end
|
269
|
+
|
270
|
+
event, payload = Experiment.published.first
|
271
|
+
refute_nil event
|
272
|
+
refute_nil payload
|
273
|
+
|
274
|
+
assert_equal :mismatch, event
|
275
|
+
assert_equal 'foo', payload[:control][:exception][:message]
|
276
|
+
assert_equal 'RuntimeError', payload[:control][:exception][:class]
|
277
|
+
refute_nil payload[:control][:exception][:backtrace]
|
278
|
+
assert_equal 'bar', payload[:candidate][:exception][:message]
|
279
|
+
assert_equal 'RuntimeError', payload[:candidate][:exception][:class]
|
280
|
+
refute_nil payload[:candidate][:exception][:backtrace]
|
281
|
+
end
|
260
282
|
end
|
data/test/dat_science_test.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dat-science
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,40 +10,40 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2014-03-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: minitest
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
|
-
- -
|
20
|
+
- - ~>
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
22
|
+
version: 5.0.8
|
23
23
|
type: :development
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
none: false
|
27
27
|
requirements:
|
28
|
-
- -
|
28
|
+
- - ~>
|
29
29
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
30
|
+
version: 5.0.8
|
31
31
|
- !ruby/object:Gem::Dependency
|
32
32
|
name: mocha
|
33
33
|
requirement: !ruby/object:Gem::Requirement
|
34
34
|
none: false
|
35
35
|
requirements:
|
36
|
-
- -
|
36
|
+
- - ~>
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
38
|
+
version: 0.14.0
|
39
39
|
type: :development
|
40
40
|
prerelease: false
|
41
41
|
version_requirements: !ruby/object:Gem::Requirement
|
42
42
|
none: false
|
43
43
|
requirements:
|
44
|
-
- -
|
44
|
+
- - ~>
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 0.14.0
|
47
47
|
description: Gradually test, measure, and track refactored code.
|
48
48
|
email:
|
49
49
|
- jbarnette@github.com
|