scientist 0.0.3 → 0.0.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 78972aec5f81213b70c0d80841e4608ab6f933c4
4
- data.tar.gz: bcdb905245beab4a89e6a2ffe63da983197ddc26
3
+ metadata.gz: 89ebfbe0cbc1c0de8e8247f59fac21a7ac547e96
4
+ data.tar.gz: 50b3ee5ed061c80af3ed1e10098acab524d82dc5
5
5
  SHA512:
6
- metadata.gz: 9428bf617e6c74d85b44b55b5a4e826177cf34dca77f1de80eb25143e13ee78f849873619d1d9362d65d79eb26939686062d4ea34531faa61ff152e70a343e62
7
- data.tar.gz: 8a9fdcdf0cb10bc41b65cea0c725450f508890829c60249e0effdc5d09c15ca420f37db31f3968b64d5fe27a76290266b1f90baa1f79cec015c78ddac72e609d
6
+ metadata.gz: 1c35cd1392cca5da51729d825f9b4b9e58ed3439247141438070ad5137de59600549e12a3eff4a0ed73ee19fa220de4cd66fe37eb2207df5f1a200b7cde532d4
7
+ data.tar.gz: 80cf84df09f69c2fb3f3f51b6fd72ca3a7d569b0a57c6d5a7dc48d48a597b7add4bbebea149d5dc2bef3aedc9ce5e02897c24285596d9cbbd912f68ec1919e0f
data/README.md CHANGED
@@ -144,6 +144,25 @@ end
144
144
 
145
145
  The `widget-permissions` and `widget-destruction` experiments will both have a `:widget` key in their contexts.
146
146
 
147
+ ### Expensive setup
148
+
149
+ If an experiment requires expensive setup that should only occur when the experiment is going to be run, define it with the `before_run` method:
150
+
151
+ ```ruby
152
+ # Code under test modifies this in-place. We want to copy it for the
153
+ # candidate code, but only when needed:
154
+ value_for_original_code = big_object
155
+ value_for_new_code = nil
156
+
157
+ science "expensive-but-worthwhile" do |e|
158
+ e.before_run do
159
+ value_for_new_code = big_object.deep_copy
160
+ end
161
+ e.use { original_code(value_for_original_code) }
162
+ e.try { new_code(value_for_new_code) }
163
+ end
164
+ ```
165
+
147
166
  ### Keeping it clean
148
167
 
149
168
  Sometimes you don't want to store the full value for later analysis. For example, an experiment may return `User` instances, but when researching a mismatch, all you care about is the logins. You can define how to clean these values in an experiment:
@@ -410,7 +429,7 @@ end
410
429
 
411
430
  ## Hacking
412
431
 
413
- Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are installed automatically. Science requires Ruby 2.1.
432
+ Be on a Unixy box. Make sure a modern Bundler is available. `script/test` runs the unit tests. All development dependencies are installed automatically. Science requires Ruby 1.9 or newer.
414
433
 
415
434
  ## Maintainers
416
435
 
@@ -15,8 +15,33 @@ module Scientist::Experiment
15
15
 
16
16
  # A mismatch, raised when raise_on_mismatches is enabled.
17
17
  class MismatchError < StandardError
18
+ attr_reader :name, :result
19
+
18
20
  def initialize(name, result)
19
- super "#{name}: control #{result.control.inspect}, candidates #{result.candidates.map(&:inspect)}"
21
+ @name = name
22
+ @result = result
23
+ super "experiment '#{name}' observations mismatched"
24
+ end
25
+
26
+ # The default formatting is nearly unreadable, so make it useful.
27
+ #
28
+ # The assumption here is that errors raised in a test environment are
29
+ # printed out as strings, rather than using #inspect.
30
+ def to_s
31
+ super + ":\n" +
32
+ format_observation(result.control) + "\n" +
33
+ result.candidates.map { |candidate| format_observation(candidate) }.join("\n") +
34
+ "\n"
35
+ end
36
+
37
+ def format_observation(observation)
38
+ observation.name + ":\n" +
39
+ if observation.raised?
40
+ observation.exception.inspect.prepend(" ") + "\n" +
41
+ observation.exception.backtrace.map { |line| line.prepend(" ") }.join("\n")
42
+ else
43
+ observation.value.inspect.prepend(" ")
44
+ end
20
45
  end
21
46
  end
22
47
 
@@ -42,6 +67,16 @@ module Scientist::Experiment
42
67
  base.extend RaiseOnMismatch
43
68
  end
44
69
 
70
+ # Define a block of code to run before an experiment begins, if the experiment
71
+ # is enabled.
72
+ #
73
+ # The block takes no arguments.
74
+ #
75
+ # Returns the configured block.
76
+ def before_run(&block)
77
+ @_scientist_before_run = block
78
+ end
79
+
45
80
  # A Hash of behavior blocks, keyed by String name. Register behavior blocks
46
81
  # with the `try` and `use` methods.
47
82
  def behaviors
@@ -158,7 +193,13 @@ module Scientist::Experiment
158
193
  raise Scientist::BehaviorMissing.new(self, name)
159
194
  end
160
195
 
161
- return block.call unless should_experiment_run?
196
+ unless should_experiment_run?
197
+ return block.call
198
+ end
199
+
200
+ if @_scientist_before_run
201
+ @_scientist_before_run.call
202
+ end
162
203
 
163
204
  observations = []
164
205
 
@@ -177,15 +218,15 @@ module Scientist::Experiment
177
218
  raised :publish, ex
178
219
  end
179
220
 
180
- if control.raised?
181
- raise control.exception
182
- end
183
-
184
221
  if self.class.raise_on_mismatches? && result.mismatched?
185
- raise MismatchError.new(name, result)
222
+ raise MismatchError.new(self.name, result)
186
223
  end
187
224
 
188
- control.value
225
+ if control.raised?
226
+ raise control.exception
227
+ else
228
+ control.value
229
+ end
189
230
  end
190
231
 
191
232
  # Define a block that determines whether or not the experiment should run.
@@ -1,3 +1,3 @@
1
1
  module Scientist
2
- VERSION = "0.0.3"
2
+ VERSION = "0.0.4"
3
3
  end
@@ -392,7 +392,112 @@ describe Scientist::Experiment do
392
392
  @ex.use { "fine" }
393
393
  @ex.try { "not fine" }
394
394
 
395
+ assert_equal "fine", @ex.run
396
+ end
397
+
398
+ it "raises a mismatch error if the control raises and candidate doesn't" do
399
+ Fake.raise_on_mismatches = true
400
+ @ex.use { raise "control" }
401
+ @ex.try { "candidate" }
402
+ # assert_raises(Scientist::Experiment::MismatchError) { @ex.run }
403
+ @ex.run
404
+ end
405
+
406
+ it "raises a mismatch error if the candidate raises and the control doesn't" do
407
+ Fake.raise_on_mismatches = true
408
+ @ex.use { "control" }
409
+ @ex.try { raise "candidate" }
410
+ assert_raises(Scientist::Experiment::MismatchError) { @ex.run }
411
+ end
412
+
413
+ describe "MismatchError" do
414
+ before do
415
+ Fake.raise_on_mismatches = true
416
+ @ex.use { :foo }
417
+ @ex.try { :bar }
418
+ begin
419
+ @ex.run
420
+ rescue Scientist::Experiment::MismatchError => e
421
+ @mismatch = e
422
+ end
423
+ assert @mismatch
424
+ end
425
+
426
+ it "has the name of the experiment" do
427
+ assert_equal @ex.name, @mismatch.name
428
+ end
429
+
430
+ it "includes the experiments' results" do
431
+ assert_equal @ex.published_result, @mismatch.result
432
+ end
433
+
434
+ it "formats nicely as a string" do
435
+ assert_equal <<-STR, @mismatch.to_s
436
+ experiment 'experiment' observations mismatched:
437
+ control:
438
+ :foo
439
+ candidate:
440
+ :bar
441
+ STR
442
+ end
443
+
444
+ it "includes the backtrace when an observation raises" do
445
+ mismatch = nil
446
+ ex = Fake.new
447
+ ex.use { "value" }
448
+ ex.try { raise "error" }
449
+
450
+ begin
451
+ ex.run
452
+ rescue Scientist::Experiment::MismatchError => e
453
+ mismatch = e
454
+ end
455
+
456
+ # Should look like this:
457
+ # experiment 'experiment' observations mismatched:
458
+ # control:
459
+ # "value"
460
+ # candidate:
461
+ # #<RuntimeError: error>
462
+ # test/scientist/experiment_test.rb:447:in `block (5 levels) in <top (required)>'
463
+ # ... (more backtrace)
464
+ lines = mismatch.to_s.split("\n")
465
+ assert_equal "control:", lines[1]
466
+ assert_equal " \"value\"", lines[2]
467
+ assert_equal "candidate:", lines[3]
468
+ assert_equal " #<RuntimeError: error>", lines[4]
469
+ assert_match %r( test/scientist/experiment_test.rb:\d+:in `block), lines[5]
470
+ end
471
+ end
472
+ end
473
+
474
+ describe "before run block" do
475
+ it "runs when an experiment is enabled" do
476
+ control_ok = candidate_ok = false
477
+ before = false
478
+ @ex.before_run { before = true }
479
+ @ex.use { control_ok = before }
480
+ @ex.try { candidate_ok = before }
481
+
395
482
  @ex.run
483
+
484
+ assert before, "before_run should have run"
485
+ assert control_ok, "control should have run after before_run"
486
+ assert candidate_ok, "candidate should have run after before_run"
487
+ end
488
+
489
+ it "does not run when an experiment is disabled" do
490
+ before = false
491
+
492
+ def @ex.enabled?
493
+ false
494
+ end
495
+ @ex.before_run { before = true }
496
+ @ex.use { "value" }
497
+ @ex.try { "value" }
498
+ @ex.run
499
+
500
+ refute before, "before_run should not have run"
396
501
  end
397
502
  end
398
503
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scientist
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Barnette
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-12-30 00:00:00.000000000 Z
12
+ date: 2015-02-18 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: minitest