async_experiments 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENCE.txt +21 -0
- data/README.md +127 -0
- data/lib/async_experiments/candidate_worker.rb +29 -0
- data/lib/async_experiments/experiment_control.rb +27 -0
- data/lib/async_experiments/experiment_error_worker.rb +14 -0
- data/lib/async_experiments/experiment_result.rb +90 -0
- data/lib/async_experiments/experiment_result_worker.rb +36 -0
- data/lib/async_experiments/util.rb +20 -0
- data/lib/async_experiments/version.rb +3 -0
- data/lib/async_experiments.rb +51 -0
- data/spec/async_experiments_spec.rb +49 -0
- data/spec/candidate_worker_spec.rb +96 -0
- data/spec/examples.txt +37 -0
- data/spec/experiment_control_spec.rb +60 -0
- data/spec/experiment_error_worker_spec.rb +36 -0
- data/spec/experiment_result_spec.rb +136 -0
- data/spec/experiment_result_worker_spec.rb +106 -0
- data/spec/spec_helper.rb +101 -0
- data/spec/util_spec.rb +78 -0
- metadata +184 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 73d448d952cbbee2567c807bafd327fead90ac38
|
4
|
+
data.tar.gz: 47f6da2a76c6f75c42ea8904c72421164bc77c3c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c3d83b96b7143a1594c36d3d29b3ba77e6dd65409a35201dae68059cfdc8e2ffe4e03e28b6805d58251a82340984daee93eaf1ec40c03c820caaaf0e85b28ca2
|
7
|
+
data.tar.gz: a8932415d200081e7a6176175cbd29372a5e91463cdc040e5d72a5424939af9e10176ef149efa891868af3e4c84cc009ffe06fd5bf99f5ebdf55ee24b47c0100
|
data/LICENCE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Crown Copyright (Government Digital Service)
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
# Asynchronous Experiments
|
2
|
+
|
3
|
+
Similar to GitHub Scientist, but uses Sidekiq (and its Redis connection pool)
|
4
|
+
to run control and candidate branches of experiments in parallel, storing the
|
5
|
+
output for later review.
|
6
|
+
|
7
|
+
Provides helpers to assist with rendering the output from the comparison.
|
8
|
+
|
9
|
+
## IMPORTANT NOTE ABOUT SIDEKIQ
|
10
|
+
|
11
|
+
This gem expects Sidekiq to be included, but does not list it as a gem dependency.
|
12
|
+
|
13
|
+
This is because GOV.UK uses `govuk_sidekiq`, a gem which automatically configures
|
14
|
+
our apps with some standard Sidekiq configurations. We haven't added that to the
|
15
|
+
gemspec as other organisations may want to use other methods to configure Sidekiq.
|
16
|
+
|
17
|
+
The gem also assumes access to statsd for reporting purposes.
|
18
|
+
|
19
|
+
## Technical documentation
|
20
|
+
|
21
|
+
Example usage:
|
22
|
+
```
|
23
|
+
require "async_experiments/experiment_control"
|
24
|
+
|
25
|
+
class ContentItemsController < ApplicationController
|
26
|
+
include AsyncExperiments::ExperimentControl
|
27
|
+
|
28
|
+
def linkables
|
29
|
+
candidate = {
|
30
|
+
worker: LinkablesCandidate,
|
31
|
+
args: [
|
32
|
+
query_params.fetch(:document_type),
|
33
|
+
],
|
34
|
+
}
|
35
|
+
|
36
|
+
presented = experiment_control(:linkables, candidate: candidate) {
|
37
|
+
Queries::GetContentCollection.new(
|
38
|
+
document_type: query_params.fetch(:document_type),
|
39
|
+
fields: %w(
|
40
|
+
title
|
41
|
+
content_id
|
42
|
+
publication_state
|
43
|
+
base_path
|
44
|
+
internal_name
|
45
|
+
),
|
46
|
+
pagination: NullPagination.new
|
47
|
+
).call
|
48
|
+
}
|
49
|
+
|
50
|
+
render json: presented
|
51
|
+
end
|
52
|
+
end
|
53
|
+
```
|
54
|
+
|
55
|
+
```
|
56
|
+
require "async_experiments/candidate_worker"
|
57
|
+
|
58
|
+
class LinkablesCandidate < AsyncExperiments::CandidateWorker
|
59
|
+
def perform(document_type, experiment)
|
60
|
+
experiment_candidate(experiment) do
|
61
|
+
Queries::GetLinkables.new(
|
62
|
+
document_type: document_type,
|
63
|
+
).call
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
```
|
68
|
+
|
69
|
+
```
|
70
|
+
class DebugController < ApplicationController
|
71
|
+
skip_before_action :require_signin_permission!
|
72
|
+
before_action :validate_experiment_name, only: [:experiment]
|
73
|
+
|
74
|
+
def experiment
|
75
|
+
@mismatched_responses = AsyncExperiments.get_experiment_data(params[:experiment])
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def validate_experiment_name
|
81
|
+
raise "Experiment names don't contain `:`" if params[:experiment].include?(":")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
```
|
85
|
+
|
86
|
+
```
|
87
|
+
<ul>
|
88
|
+
<% @mismatched_responses.each_with_index do |mismatch, i| %>
|
89
|
+
<li>
|
90
|
+
<ul>
|
91
|
+
<li><a href="#missing-#{i}">Missing</a></li>
|
92
|
+
<li><a href="#extra-#{i}">Extra</a></li>
|
93
|
+
<li><a href="#changed-#{i}">Changed</a></li>
|
94
|
+
</ul>
|
95
|
+
|
96
|
+
<h3 id="missing-#{i}">Missing from candidate</h3>
|
97
|
+
<% mismatch[:missing].each do |entry| %>
|
98
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
99
|
+
<% end %>
|
100
|
+
|
101
|
+
<h3 id="extra-#{i}">Extra in candidate</h3>
|
102
|
+
<% mismatch[:extra].each do |entry| %>
|
103
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
104
|
+
<% end %>
|
105
|
+
|
106
|
+
<h3 id="changed-#{i}">Changed in candidate</h3>
|
107
|
+
<% mismatch[:changed].each do |entry| %>
|
108
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
109
|
+
<% end %>
|
110
|
+
</li>
|
111
|
+
<% end %>
|
112
|
+
</ul>
|
113
|
+
```
|
114
|
+
|
115
|
+
```
|
116
|
+
statsd_client = Statsd.new("localhost")
|
117
|
+
statsd_client.namespace = "govuk.app.publishing-api"
|
118
|
+
AsyncExperiments.statsd = statsd_client
|
119
|
+
```
|
120
|
+
|
121
|
+
## Licence
|
122
|
+
|
123
|
+
[MIT License](LICENCE)
|
124
|
+
|
125
|
+
## Versioning policy
|
126
|
+
|
127
|
+
See https://github.com/alphagov/styleguides/blob/master/rubygems.md#versioning
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "async_experiments/experiment_result_worker"
|
2
|
+
require "async_experiments/experiment_error_worker"
|
3
|
+
|
4
|
+
module AsyncExperiments
|
5
|
+
class CandidateWorker
|
6
|
+
include Sidekiq::Worker
|
7
|
+
|
8
|
+
sidekiq_options queue: :experiments
|
9
|
+
|
10
|
+
def experiment_candidate(experiment_config)
|
11
|
+
experiment = experiment_config.symbolize_keys
|
12
|
+
|
13
|
+
start_time = Time.now
|
14
|
+
run_output = yield
|
15
|
+
duration = (Time.now - start_time).to_f
|
16
|
+
ExperimentResultWorker.perform_async(experiment[:name], experiment[:id], run_output, duration, :candidate)
|
17
|
+
|
18
|
+
run_output
|
19
|
+
rescue StandardError => exception
|
20
|
+
if ENV["RAISE_EXPERIMENT_ERRORS"]
|
21
|
+
raise exception
|
22
|
+
else
|
23
|
+
backtrace = exception.backtrace
|
24
|
+
backtrace.unshift(exception.inspect)
|
25
|
+
ExperimentErrorWorker.perform_async(experiment[:name], backtrace.join("\n"))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "async_experiments/experiment_result_worker"
|
2
|
+
|
3
|
+
module AsyncExperiments
|
4
|
+
module ExperimentControl
|
5
|
+
def experiment_control(name, candidate:)
|
6
|
+
start_time = Time.now
|
7
|
+
run_output = yield
|
8
|
+
duration = (Time.now - start_time).to_f
|
9
|
+
|
10
|
+
id = SecureRandom.uuid
|
11
|
+
|
12
|
+
if run_output.class == Enumerator
|
13
|
+
run_output = run_output.to_a
|
14
|
+
end
|
15
|
+
|
16
|
+
ExperimentResultWorker.perform_async(name, id, run_output, duration, :control)
|
17
|
+
|
18
|
+
candidate_worker = candidate.fetch(:worker)
|
19
|
+
candidate_worker.perform_async(*candidate.fetch(:args),
|
20
|
+
name: name,
|
21
|
+
id: id,
|
22
|
+
)
|
23
|
+
|
24
|
+
run_output
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module AsyncExperiments
|
2
|
+
class ExperimentErrorWorker
|
3
|
+
include Sidekiq::Worker
|
4
|
+
|
5
|
+
sidekiq_options queue: :experiments
|
6
|
+
|
7
|
+
def perform(experiment_name, exception_string)
|
8
|
+
Sidekiq.redis do |redis|
|
9
|
+
AsyncExperiments.statsd.increment("experiments.#{experiment_name}.exceptions")
|
10
|
+
redis.rpush("experiments:#{experiment_name}:exceptions", exception_string)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require "json"
|
2
|
+
require "hashdiff"
|
3
|
+
require "async_experiments/util"
|
4
|
+
|
5
|
+
module AsyncExperiments
|
6
|
+
class ExperimentResult
|
7
|
+
def initialize(name, id, type, redis, statsd, run_output = nil, duration = nil)
|
8
|
+
@name = name
|
9
|
+
@key = "#{name}:#{id}"
|
10
|
+
@redis = redis
|
11
|
+
@statsd = statsd
|
12
|
+
@type = type
|
13
|
+
@run_output = run_output
|
14
|
+
@duration = duration
|
15
|
+
|
16
|
+
if Util.blank?(run_output) || Util.blank?(duration)
|
17
|
+
redis_data = data_from_redis
|
18
|
+
|
19
|
+
if redis_data
|
20
|
+
@run_output ||= redis_data.fetch(:run_output)
|
21
|
+
@duration ||= redis_data.fetch(:duration)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
attr_reader :key, :run_output, :duration
|
27
|
+
|
28
|
+
def store_run_output
|
29
|
+
redis.set("experiments:#{key}:#{type}", {
|
30
|
+
run_output: run_output,
|
31
|
+
duration: duration,
|
32
|
+
}.to_json)
|
33
|
+
end
|
34
|
+
|
35
|
+
def process_run_output(candidate)
|
36
|
+
variation = HashDiff.diff(sort(self.run_output), sort(candidate.run_output))
|
37
|
+
report_data(variation, candidate)
|
38
|
+
redis.del("experiments:#{key}:candidate")
|
39
|
+
end
|
40
|
+
|
41
|
+
def control?
|
42
|
+
type == :control
|
43
|
+
end
|
44
|
+
|
45
|
+
def candidate?
|
46
|
+
type == :candidate
|
47
|
+
end
|
48
|
+
|
49
|
+
def available?
|
50
|
+
Util.present?(run_output) && Util.present?(duration)
|
51
|
+
end
|
52
|
+
|
53
|
+
protected
|
54
|
+
|
55
|
+
attr_reader :redis, :statsd, :type, :name
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def data_from_redis
|
60
|
+
redis_data = redis.get("experiments:#{key}:#{type}")
|
61
|
+
|
62
|
+
if Util.present?(redis_data)
|
63
|
+
Util.deep_symbolize_keys(JSON.parse(redis_data))
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def report_data(variation, candidate)
|
68
|
+
statsd.timing("experiments.#{name}.control", self.duration)
|
69
|
+
statsd.timing("experiments.#{name}.candidate", candidate.duration)
|
70
|
+
|
71
|
+
if variation != []
|
72
|
+
statsd.increment("experiments.#{name}.mismatches")
|
73
|
+
redis.rpush("experiments:#{name}:mismatches", JSON.dump(variation))
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def sort(object)
|
78
|
+
case object
|
79
|
+
when Array
|
80
|
+
object.sort_by(&:object_id)
|
81
|
+
when Hash
|
82
|
+
object.each_with_object({}) { |(key, value), hash|
|
83
|
+
hash[key] = sort(value)
|
84
|
+
}
|
85
|
+
else
|
86
|
+
object
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "async_experiments/experiment_result"
|
2
|
+
|
3
|
+
module AsyncExperiments
|
4
|
+
class ExperimentResultWorker
|
5
|
+
include Sidekiq::Worker
|
6
|
+
|
7
|
+
sidekiq_options queue: :experiments
|
8
|
+
|
9
|
+
LOCK_TIMEOUT = 60
|
10
|
+
|
11
|
+
def perform(name, id, run_output, duration, type)
|
12
|
+
type = type.to_sym
|
13
|
+
|
14
|
+
Sidekiq.redis do |redis|
|
15
|
+
this_branch = ExperimentResult.new(name, id, type, redis, statsd, run_output, duration)
|
16
|
+
|
17
|
+
if this_branch.control?
|
18
|
+
candidate = ExperimentResult.new(name, id, :candidate, redis, statsd)
|
19
|
+
if candidate.available?
|
20
|
+
this_branch.process_run_output(candidate)
|
21
|
+
else
|
22
|
+
self.class.perform_in(5, name, id, run_output, duration, type)
|
23
|
+
end
|
24
|
+
elsif this_branch.candidate?
|
25
|
+
this_branch.store_run_output
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def statsd
|
33
|
+
AsyncExperiments.statsd
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module AsyncExperiments
|
2
|
+
module Util
|
3
|
+
def self.present?(object)
|
4
|
+
!self.blank?(object)
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.blank?(object)
|
8
|
+
object.nil? || (object.respond_to?(:empty?) && object.empty?)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.deep_symbolize_keys(hash)
|
12
|
+
return hash unless hash.is_a?(Hash)
|
13
|
+
|
14
|
+
hash.each_with_object({}) do |(key, value), new_hash|
|
15
|
+
key = key.respond_to?(:to_sym) ? key.to_sym : key
|
16
|
+
new_hash[key] = self.deep_symbolize_keys(value)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require "json"
|
2
|
+
require "securerandom"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
module AsyncExperiments
|
6
|
+
def self.statsd
|
7
|
+
@statsd
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.statsd=(statsd)
|
11
|
+
@statsd = statsd
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.get_experiment_data(experiment_name)
|
15
|
+
mismatched_responses = Sidekiq.redis { |redis|
|
16
|
+
redis.lrange("experiments:#{experiment_name}:mismatches", 0, -1)
|
17
|
+
}
|
18
|
+
|
19
|
+
mismatched_responses.map { |json|
|
20
|
+
parsed = JSON.parse(json)
|
21
|
+
|
22
|
+
missing, other = parsed.partition {|(operator, _, _)|
|
23
|
+
operator == "-"
|
24
|
+
}
|
25
|
+
|
26
|
+
extra, changed = other.partition {|(operator, _, _)|
|
27
|
+
operator == "+"
|
28
|
+
}
|
29
|
+
|
30
|
+
missing_entries, extra_entries = self.fix_ordering_issues(
|
31
|
+
missing.map(&:last),
|
32
|
+
extra.map(&:last),
|
33
|
+
)
|
34
|
+
|
35
|
+
{
|
36
|
+
missing: missing_entries,
|
37
|
+
extra: extra_entries,
|
38
|
+
changed: changed.map(&:last),
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.fix_ordering_issues(missing_entries, extra_entries)
|
44
|
+
duplicate_entries = missing_entries & extra_entries
|
45
|
+
|
46
|
+
missing_entries = missing_entries.reject { |entry| duplicate_entries.include?(entry) }
|
47
|
+
extra_entries = extra_entries.reject { |entry| duplicate_entries.include?(entry) }
|
48
|
+
|
49
|
+
[missing_entries, extra_entries]
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "json"
|
3
|
+
require "async_experiments"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments do
|
6
|
+
describe ".get_experiment_data(experiment_name)" do
|
7
|
+
let(:name) { "some_experiment" }
|
8
|
+
|
9
|
+
let(:experiment_results) {
|
10
|
+
[
|
11
|
+
JSON.dump([
|
12
|
+
["-", "[1]", {same_key: 1, different_key: 2}],
|
13
|
+
["+", "[2]", {same_key: 1, different_key: 3}],
|
14
|
+
["+", "[3]", "Extra element"],
|
15
|
+
["-", "[4]", "Missing element"],
|
16
|
+
["~", "[5]", "Changed element"],
|
17
|
+
["-", "[3]", {moved_complex_object: 1}],
|
18
|
+
["+", "[6]", {moved_complex_object: 1}],
|
19
|
+
])
|
20
|
+
]
|
21
|
+
}
|
22
|
+
|
23
|
+
let(:redis) { double(:redis) }
|
24
|
+
|
25
|
+
before do
|
26
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
27
|
+
allow(redis).to receive(:lrange).with("experiments:#{name}:mismatches", 0, -1)
|
28
|
+
.and_return(experiment_results)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "partitions and resorts experiment results for useful output" do
|
32
|
+
results = described_class.get_experiment_data(name)
|
33
|
+
|
34
|
+
expect(results).to eq([
|
35
|
+
missing: [
|
36
|
+
{"same_key" => 1, "different_key" => 2},
|
37
|
+
"Missing element",
|
38
|
+
],
|
39
|
+
extra: [
|
40
|
+
{"same_key" => 1, "different_key" => 3},
|
41
|
+
"Extra element",
|
42
|
+
],
|
43
|
+
changed: [
|
44
|
+
"Changed element",
|
45
|
+
],
|
46
|
+
])
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/candidate_worker"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::CandidateWorker do
|
6
|
+
let(:name) { "some_experiment" }
|
7
|
+
let(:id) { SecureRandom.uuid }
|
8
|
+
|
9
|
+
let(:run_output) { [{some: "output"}] }
|
10
|
+
|
11
|
+
class TestWorker < AsyncExperiments::CandidateWorker
|
12
|
+
def perform(run_output, experiment_config)
|
13
|
+
experiment_candidate(experiment_config) do
|
14
|
+
Timecop.travel(Time.now + 1.5)
|
15
|
+
run_output
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
subject { TestWorker.new }
|
21
|
+
|
22
|
+
it "uses the 'experiments' queue" do
|
23
|
+
Sidekiq::Testing.fake! do
|
24
|
+
TestWorker.perform_async
|
25
|
+
|
26
|
+
expect(TestWorker.jobs.size).to eq(1)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
it "returns the control run output" do
|
32
|
+
output = subject.perform(run_output,
|
33
|
+
name: name,
|
34
|
+
id: id,
|
35
|
+
)
|
36
|
+
|
37
|
+
expect(output).to eq(run_output)
|
38
|
+
end
|
39
|
+
|
40
|
+
describe "#experiment_candidate(experiment_config)" do
|
41
|
+
it "triggers an ExperimentResultWorker with the candidate output and duration" do
|
42
|
+
expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
|
43
|
+
.with(name, id, run_output, instance_of(Float), :candidate)
|
44
|
+
|
45
|
+
subject.perform(run_output,
|
46
|
+
name: name,
|
47
|
+
id: id,
|
48
|
+
)
|
49
|
+
end
|
50
|
+
|
51
|
+
context "when experiment errors are being raised" do
|
52
|
+
around do |example|
|
53
|
+
setting = ENV["RAISE_EXPERIMENT_ERRORS"]
|
54
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
|
55
|
+
|
56
|
+
example.run
|
57
|
+
|
58
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = setting
|
59
|
+
end
|
60
|
+
|
61
|
+
it "re-raises experiment errors" do
|
62
|
+
allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
|
63
|
+
|
64
|
+
expect {
|
65
|
+
subject.perform(run_output,
|
66
|
+
"name" => name,
|
67
|
+
"id" => id,
|
68
|
+
)
|
69
|
+
}.to raise_error(Exception, "Test exception")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
context "when experiment errors are being quietly reported" do
|
74
|
+
around do |example|
|
75
|
+
setting = ENV["RAISE_EXPERIMENT_ERRORS"]
|
76
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = nil
|
77
|
+
|
78
|
+
example.run
|
79
|
+
|
80
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = setting
|
81
|
+
end
|
82
|
+
|
83
|
+
it "re-raises experiment errors" do
|
84
|
+
allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
|
85
|
+
|
86
|
+
expect(AsyncExperiments::ExperimentErrorWorker).to receive(:perform_async)
|
87
|
+
.with(name, instance_of(String))
|
88
|
+
|
89
|
+
subject.perform(run_output,
|
90
|
+
name: name,
|
91
|
+
id: id,
|
92
|
+
)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
data/spec/examples.txt
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
example_id | status | run_time |
|
2
|
+
------------------------------------------------ | ------ | --------------- |
|
3
|
+
./spec/async_experiments_spec.rb[1:1:1] | passed | 0.00062 seconds |
|
4
|
+
./spec/candidate_worker_spec.rb[1:1] | passed | 0.00023 seconds |
|
5
|
+
./spec/candidate_worker_spec.rb[1:2] | passed | 0.00053 seconds |
|
6
|
+
./spec/candidate_worker_spec.rb[1:3:1] | passed | 0.00069 seconds |
|
7
|
+
./spec/candidate_worker_spec.rb[1:3:2:1] | passed | 0.00209 seconds |
|
8
|
+
./spec/candidate_worker_spec.rb[1:3:3:1] | passed | 0.0008 seconds |
|
9
|
+
./spec/experiment_control_spec.rb[1:1:1] | passed | 0.00089 seconds |
|
10
|
+
./spec/experiment_control_spec.rb[1:1:2] | passed | 0.00117 seconds |
|
11
|
+
./spec/experiment_control_spec.rb[1:1:3] | passed | 0.00061 seconds |
|
12
|
+
./spec/experiment_error_worker_spec.rb[1:1] | passed | 0.00266 seconds |
|
13
|
+
./spec/experiment_error_worker_spec.rb[1:2] | passed | 0.00056 seconds |
|
14
|
+
./spec/experiment_error_worker_spec.rb[1:3] | passed | 0.0077 seconds |
|
15
|
+
./spec/experiment_result_spec.rb[1:1:1] | passed | 0.00042 seconds |
|
16
|
+
./spec/experiment_result_spec.rb[1:2:1] | passed | 0.00043 seconds |
|
17
|
+
./spec/experiment_result_spec.rb[1:3:1] | passed | 0.00071 seconds |
|
18
|
+
./spec/experiment_result_spec.rb[1:3:2] | passed | 0.00045 seconds |
|
19
|
+
./spec/experiment_result_spec.rb[1:3:3:1] | passed | 0.00055 seconds |
|
20
|
+
./spec/experiment_result_spec.rb[1:3:3:2] | passed | 0.00048 seconds |
|
21
|
+
./spec/experiment_result_spec.rb[1:3:4:1] | passed | 0.00043 seconds |
|
22
|
+
./spec/experiment_result_spec.rb[1:3:4:2] | passed | 0.00058 seconds |
|
23
|
+
./spec/experiment_result_spec.rb[1:4:1:1] | passed | 0.00037 seconds |
|
24
|
+
./spec/experiment_result_spec.rb[1:4:2:1:1] | passed | 0.00095 seconds |
|
25
|
+
./spec/experiment_result_spec.rb[1:4:2:1:2] | passed | 0.00055 seconds |
|
26
|
+
./spec/experiment_result_spec.rb[1:4:2:2:1] | passed | 0.0005 seconds |
|
27
|
+
./spec/experiment_result_worker_spec.rb[1:1] | passed | 0.00048 seconds |
|
28
|
+
./spec/experiment_result_worker_spec.rb[1:2:1] | passed | 0.00175 seconds |
|
29
|
+
./spec/experiment_result_worker_spec.rb[1:3:1:1] | passed | 0.0013 seconds |
|
30
|
+
./spec/experiment_result_worker_spec.rb[1:3:2:1] | passed | 0.00235 seconds |
|
31
|
+
./spec/experiment_result_worker_spec.rb[1:3:2:2] | passed | 0.0025 seconds |
|
32
|
+
./spec/util_spec.rb[1:1:1] | passed | 0.00021 seconds |
|
33
|
+
./spec/util_spec.rb[1:2:1] | passed | 0.00016 seconds |
|
34
|
+
./spec/util_spec.rb[1:3:1] | passed | 0.00014 seconds |
|
35
|
+
./spec/util_spec.rb[1:3:2] | passed | 0.00016 seconds |
|
36
|
+
./spec/util_spec.rb[1:3:3] | passed | 0.00012 seconds |
|
37
|
+
./spec/util_spec.rb[1:3:4] | passed | 0.00011 seconds |
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/experiment_control"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::ExperimentControl do
|
6
|
+
let(:name) { :some_experiment }
|
7
|
+
let(:id) { double(:id) }
|
8
|
+
|
9
|
+
let(:candidate_args) { [1, 2, 3] }
|
10
|
+
let(:candidate_worker) { double(:candidate_worker, perform_async: nil) }
|
11
|
+
let(:candidate_config) {
|
12
|
+
{
|
13
|
+
worker: candidate_worker,
|
14
|
+
args: candidate_args,
|
15
|
+
}
|
16
|
+
}
|
17
|
+
|
18
|
+
let(:run_output) { [{some: "output"}] }
|
19
|
+
|
20
|
+
class TestClass
|
21
|
+
include AsyncExperiments::ExperimentControl
|
22
|
+
|
23
|
+
def call(name, run_output, candidate_config)
|
24
|
+
experiment_control(name, candidate: candidate_config) do
|
25
|
+
Timecop.travel(Time.now + 1.5)
|
26
|
+
run_output
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
subject { TestClass.new }
|
32
|
+
|
33
|
+
before do
|
34
|
+
allow(SecureRandom).to receive(:uuid).and_return(id)
|
35
|
+
end
|
36
|
+
|
37
|
+
describe "#experiment_control(name, candidate: candidate_config)" do
|
38
|
+
it "triggers an ExperimentResultWorker with the control output and duration" do
|
39
|
+
expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
|
40
|
+
.with(name, id, run_output, instance_of(Float), :control)
|
41
|
+
|
42
|
+
subject.call(name, run_output, candidate_config)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "triggers the candidate worker with its arguments and the experiment config" do
|
46
|
+
expect(candidate_worker).to receive(:perform_async)
|
47
|
+
.with(*candidate_args,
|
48
|
+
name: name,
|
49
|
+
id: id,
|
50
|
+
)
|
51
|
+
|
52
|
+
subject.call(name, run_output, candidate_config)
|
53
|
+
end
|
54
|
+
|
55
|
+
it "returns the control run output" do
|
56
|
+
output = subject.call(name, run_output, candidate_config)
|
57
|
+
expect(output).to eq(run_output)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments"
|
3
|
+
require "async_experiments/experiment_error_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::ExperimentErrorWorker do
|
6
|
+
let(:name) { "some_experiment" }
|
7
|
+
let(:error) { "Something went wrong" }
|
8
|
+
|
9
|
+
let(:statsd) { double(:statsd, increment: nil) }
|
10
|
+
let(:redis) { double(:redis, rpush: nil) }
|
11
|
+
|
12
|
+
subject { described_class.new }
|
13
|
+
|
14
|
+
before do
|
15
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
16
|
+
AsyncExperiments.statsd = statsd
|
17
|
+
end
|
18
|
+
|
19
|
+
it "uses the 'experiments' queue" do
|
20
|
+
Sidekiq::Testing.fake! do
|
21
|
+
described_class.perform_async
|
22
|
+
|
23
|
+
expect(described_class.jobs.size).to eq(1)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it "increments the statsd error count for the experiment" do
|
28
|
+
expect(statsd).to receive(:increment).with("experiments.#{name}.exceptions")
|
29
|
+
subject.perform(name, error)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "stores the exception for later reporting" do
|
33
|
+
expect(redis).to receive(:rpush).with("experiments:#{name}:exceptions", error)
|
34
|
+
subject.perform(name, error)
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/experiment_result"
|
3
|
+
|
4
|
+
RSpec.describe AsyncExperiments::ExperimentResult do
|
5
|
+
let(:id) { SecureRandom.uuid }
|
6
|
+
let(:name) { :test_experiment }
|
7
|
+
|
8
|
+
let(:control_run_output) { "control output" }
|
9
|
+
let(:control_duration) { 10.0 }
|
10
|
+
|
11
|
+
let(:candidate_run_output) { "candidate output" }
|
12
|
+
let(:candidate_duration) { 5.0 }
|
13
|
+
|
14
|
+
let(:redis) { double(:redis, del: nil) }
|
15
|
+
let(:statsd) { double(:statsd, timing: nil, increment: nil) }
|
16
|
+
|
17
|
+
let(:control) { described_class.new(name, id, :control, redis, statsd, control_run_output, control_duration) }
|
18
|
+
let(:candidate) { described_class.new(name, id, :candidate, redis, statsd, candidate_run_output, candidate_duration) }
|
19
|
+
|
20
|
+
describe "#key" do
|
21
|
+
it "builds a key from the name and ID" do
|
22
|
+
expect(control.key).to eq("test_experiment:#{id}")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "#store_run_output" do
|
27
|
+
it "stores the branch's run output and duration" do
|
28
|
+
expect(redis).to receive(:set)
|
29
|
+
.with("experiments:#{name}:#{id}:candidate", {
|
30
|
+
run_output: candidate_run_output,
|
31
|
+
duration: candidate_duration,
|
32
|
+
}.to_json)
|
33
|
+
|
34
|
+
candidate.store_run_output
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "control#process_run_output(candidate)" do
|
39
|
+
before do
|
40
|
+
allow(redis).to receive(:rpush)
|
41
|
+
end
|
42
|
+
|
43
|
+
it "reports the control and candidate durations to statsd" do
|
44
|
+
expect(statsd).to receive(:timing)
|
45
|
+
.with("experiments.#{name}.control", control_duration)
|
46
|
+
|
47
|
+
expect(statsd).to receive(:timing)
|
48
|
+
.with("experiments.#{name}.candidate", candidate_duration)
|
49
|
+
|
50
|
+
control.process_run_output(candidate)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "deletes the candidate data from redis" do
|
54
|
+
expect(redis).to receive(:del).with("experiments:#{name}:#{id}:candidate")
|
55
|
+
|
56
|
+
control.process_run_output(candidate)
|
57
|
+
end
|
58
|
+
|
59
|
+
context "if there's variation between the outputs" do
|
60
|
+
it "increments the mismatch count in statsd" do
|
61
|
+
expect(statsd).to receive(:increment)
|
62
|
+
.with("experiments.#{name}.mismatches")
|
63
|
+
|
64
|
+
control.process_run_output(candidate)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "logs the mismatch to redis" do
|
68
|
+
expect(redis).to receive(:rpush).with(
|
69
|
+
"experiments:#{name}:mismatches",
|
70
|
+
[["~", "", "control output", "candidate output"]].to_json,
|
71
|
+
)
|
72
|
+
|
73
|
+
control.process_run_output(candidate)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context "if there's no variation" do
|
78
|
+
let(:candidate_run_output) { control_run_output }
|
79
|
+
|
80
|
+
it "does not increment the mismatch count" do
|
81
|
+
expect(statsd).not_to receive(:increment)
|
82
|
+
control.process_run_output(candidate)
|
83
|
+
end
|
84
|
+
|
85
|
+
it "does not log the mismatch to redis" do
|
86
|
+
expect(redis).not_to receive(:rpush)
|
87
|
+
control.process_run_output(candidate)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe ".new" do
|
93
|
+
let(:type) { :candidate }
|
94
|
+
|
95
|
+
context "if duration and output are provided" do
|
96
|
+
it "uses those" do
|
97
|
+
candidate = described_class.new(name, id, type, redis, statsd, "arbitrary output", 1.23)
|
98
|
+
expect(candidate.run_output).to eq("arbitrary output")
|
99
|
+
expect(candidate.duration).to eq(1.23)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
context "if duration and output are not provided" do
|
104
|
+
context "and redis has the data" do
|
105
|
+
before do
|
106
|
+
allow(redis).to receive(:get).with("experiments:#{name}:#{id}:#{type}").and_return({
|
107
|
+
run_output: candidate_run_output,
|
108
|
+
duration: candidate_duration,
|
109
|
+
}.to_json)
|
110
|
+
end
|
111
|
+
|
112
|
+
it "uses the redis data" do
|
113
|
+
candidate = described_class.new(name, id, type, redis, statsd)
|
114
|
+
expect(candidate.run_output).to eq(candidate_run_output)
|
115
|
+
expect(candidate.duration).to eq(candidate_duration)
|
116
|
+
end
|
117
|
+
|
118
|
+
it "is considered available" do
|
119
|
+
candidate = described_class.new(name, id, type, redis, statsd)
|
120
|
+
expect(candidate.available?).to eq(true)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
context "but redis does not have the data" do
|
125
|
+
before do
|
126
|
+
allow(redis).to receive(:get).and_return("")
|
127
|
+
end
|
128
|
+
|
129
|
+
it "is considered unavailable" do
|
130
|
+
missing_candidate = described_class.new(name, id, type, redis, statsd)
|
131
|
+
expect(missing_candidate.available?).to eq(false)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments"
|
3
|
+
require "async_experiments/experiment_result"
|
4
|
+
require "async_experiments/experiment_result_worker"
|
5
|
+
|
6
|
+
RSpec.describe AsyncExperiments::ExperimentResultWorker do
|
7
|
+
let(:name) { "some_experiment" }
|
8
|
+
let(:id) { SecureRandom.uuid }
|
9
|
+
|
10
|
+
let(:statsd) { double(:statsd) }
|
11
|
+
let(:redis) { double(:redis) }
|
12
|
+
let(:redis_key) { double(:redis_key) }
|
13
|
+
|
14
|
+
let(:control_output) { "control output" }
|
15
|
+
let(:control_duration) { 10.0 }
|
16
|
+
|
17
|
+
let(:candidate_output) { "candidate output" }
|
18
|
+
let(:candidate_duration) { 5.0 }
|
19
|
+
|
20
|
+
let(:control) {
|
21
|
+
double(:control,
|
22
|
+
control?: true,
|
23
|
+
candidate?: false,
|
24
|
+
key: redis_key,
|
25
|
+
)
|
26
|
+
}
|
27
|
+
let(:candidate) {
|
28
|
+
double(:candidate,
|
29
|
+
candidate?: true,
|
30
|
+
control?: false,
|
31
|
+
key: redis_key,
|
32
|
+
)
|
33
|
+
}
|
34
|
+
|
35
|
+
subject { described_class.new }
|
36
|
+
|
37
|
+
before do
|
38
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
39
|
+
AsyncExperiments.statsd = statsd
|
40
|
+
end
|
41
|
+
|
42
|
+
it "uses the 'experiments' queue" do
|
43
|
+
Sidekiq::Testing.fake! do
|
44
|
+
described_class.perform_async
|
45
|
+
|
46
|
+
expect(described_class.jobs.size).to eq(1)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "if we're the candidate" do
|
51
|
+
let(:type) { "candidate" }
|
52
|
+
|
53
|
+
it "stores the run output and duration" do
|
54
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
55
|
+
.with(name, id, type.to_sym, redis, statsd, candidate_output, candidate_duration)
|
56
|
+
.and_return(candidate)
|
57
|
+
|
58
|
+
expect(candidate).to receive(:store_run_output)
|
59
|
+
|
60
|
+
subject.perform(name, id, candidate_output, candidate_duration, type)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
context "if we're the control" do
|
65
|
+
let(:type) { "control" }
|
66
|
+
|
67
|
+
before do
|
68
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
69
|
+
.with(name, id, :control, redis, statsd, control_output, control_duration)
|
70
|
+
.and_return(control)
|
71
|
+
|
72
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
73
|
+
.with(name, id, :candidate, redis, statsd)
|
74
|
+
.and_return(candidate)
|
75
|
+
end
|
76
|
+
|
77
|
+
context "and the candidate is available" do
|
78
|
+
before do
|
79
|
+
allow(candidate).to receive(:available?).and_return(true)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "processes the run output with the candidate" do
|
83
|
+
expect(control).to receive(:process_run_output).with(candidate)
|
84
|
+
subject.perform(name, id, control_output, control_duration, type)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
context "but the candidate is unavailable" do
|
89
|
+
before do
|
90
|
+
allow(candidate).to receive(:available?).and_return(false)
|
91
|
+
allow(described_class).to receive(:perform_in)
|
92
|
+
end
|
93
|
+
|
94
|
+
it "does not process the run output" do
|
95
|
+
expect(control).not_to receive(:process_run_output)
|
96
|
+
subject.perform(name, id, control_output, control_duration, type)
|
97
|
+
end
|
98
|
+
|
99
|
+
it "schedules the job to run again later" do
|
100
|
+
args = [name, id, control_output, control_duration]
|
101
|
+
expect(described_class).to receive(:perform_in).with(5, *args, type.to_sym)
|
102
|
+
subject.perform(*args, type)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
require "byebug"
|
2
|
+
require "timecop"
|
3
|
+
require 'sidekiq/testing'
|
4
|
+
Sidekiq::Logging.logger = nil
|
5
|
+
|
6
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
|
7
|
+
|
8
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
9
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
10
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause
|
11
|
+
# this file to always be loaded, without a need to explicitly require it in any
|
12
|
+
# files.
|
13
|
+
#
|
14
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
15
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
16
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
17
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
18
|
+
# a separate helper file that requires the additional dependencies and performs
|
19
|
+
# the additional setup, and require it from the spec files that actually need
|
20
|
+
# it.
|
21
|
+
#
|
22
|
+
# The `.rspec` file also contains a few flags that are not defaults but that
|
23
|
+
# users commonly want.
|
24
|
+
#
|
25
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
26
|
+
RSpec.configure do |config|
|
27
|
+
# rspec-expectations config goes here. You can use an alternate
|
28
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
29
|
+
# assertions if you prefer.
|
30
|
+
config.expect_with :rspec do |expectations|
|
31
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
32
|
+
# and `failure_message` of custom matchers include text for helper methods
|
33
|
+
# defined using `chain`, e.g.:
|
34
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
35
|
+
# # => "be bigger than 2 and smaller than 4"
|
36
|
+
# ...rather than:
|
37
|
+
# # => "be bigger than 2"
|
38
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
39
|
+
end
|
40
|
+
|
41
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
42
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
43
|
+
config.mock_with :rspec do |mocks|
|
44
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
45
|
+
# a real object. This is generally recommended, and will default to
|
46
|
+
# `true` in RSpec 4.
|
47
|
+
mocks.verify_partial_doubles = true
|
48
|
+
end
|
49
|
+
|
50
|
+
# The settings below are suggested to provide a good initial experience
|
51
|
+
# with RSpec, but feel free to customize to your heart's content.
|
52
|
+
|
53
|
+
# These two settings work together to allow you to limit a spec run
|
54
|
+
# to individual examples or groups you care about by tagging them with
|
55
|
+
# `:focus` metadata. When nothing is tagged with `:focus`, all examples
|
56
|
+
# get run.
|
57
|
+
config.filter_run :focus
|
58
|
+
config.run_all_when_everything_filtered = true
|
59
|
+
|
60
|
+
# Allows RSpec to persist some state between runs in order to support
|
61
|
+
# the `--only-failures` and `--next-failure` CLI options. We recommend
|
62
|
+
# you configure your source control system to ignore this file.
|
63
|
+
config.example_status_persistence_file_path = "spec/examples.txt"
|
64
|
+
|
65
|
+
# Limits the available syntax to the non-monkey patched syntax that is
|
66
|
+
# recommended. For more details, see:
|
67
|
+
# - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
|
68
|
+
# - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
69
|
+
# - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
|
70
|
+
config.disable_monkey_patching!
|
71
|
+
|
72
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
73
|
+
# be too noisy due to issues in dependencies.
|
74
|
+
config.warnings = true
|
75
|
+
|
76
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
77
|
+
# file, and it's useful to allow more verbose output when running an
|
78
|
+
# individual spec file.
|
79
|
+
if config.files_to_run.one?
|
80
|
+
# Use the documentation formatter for detailed output,
|
81
|
+
# unless a formatter has already been configured
|
82
|
+
# (e.g. via a command-line flag).
|
83
|
+
config.default_formatter = 'doc'
|
84
|
+
end
|
85
|
+
|
86
|
+
# Run specs in random order to surface order dependencies. If you find an
|
87
|
+
# order dependency and want to debug it, you can fix the order by providing
|
88
|
+
# the seed, which is printed after each run.
|
89
|
+
# --seed 1234
|
90
|
+
config.order = :random
|
91
|
+
|
92
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
93
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
94
|
+
# test failures related to randomization by passing the same `--seed` value
|
95
|
+
# as the one that triggered the failure.
|
96
|
+
Kernel.srand config.seed
|
97
|
+
|
98
|
+
config.before(:each) do
|
99
|
+
Sidekiq::Worker.clear_all
|
100
|
+
end
|
101
|
+
end
|
data/spec/util_spec.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/util"
|
3
|
+
|
4
|
+
RSpec.describe AsyncExperiments::Util do
|
5
|
+
describe ".present?(object)" do
|
6
|
+
it "returns true unless empty or nil" do
|
7
|
+
expect(described_class.present?("This is a test")).to eq(true)
|
8
|
+
expect(described_class.present?([1, 2])).to eq(true)
|
9
|
+
expect(described_class.present?({a: 1})).to eq(true)
|
10
|
+
|
11
|
+
expect(described_class.present?(nil)).to eq(false)
|
12
|
+
expect(described_class.present?("")).to eq(false)
|
13
|
+
expect(described_class.present?([])).to eq(false)
|
14
|
+
expect(described_class.present?({})).to eq(false)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe ".blank?(object)" do
|
19
|
+
it "returns true if empty or nil" do
|
20
|
+
expect(described_class.blank?(nil)).to eq(true)
|
21
|
+
expect(described_class.blank?("")).to eq(true)
|
22
|
+
expect(described_class.blank?([])).to eq(true)
|
23
|
+
expect(described_class.blank?({})).to eq(true)
|
24
|
+
|
25
|
+
expect(described_class.blank?("This is a test")).to eq(false)
|
26
|
+
expect(described_class.blank?([1, 2])).to eq(false)
|
27
|
+
expect(described_class.blank?({a: 1})).to eq(false)
|
28
|
+
expect(described_class.blank?(5)).to eq(false)
|
29
|
+
expect(described_class.blank?(5.5)).to eq(false)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe ".deep_symbolize_keys(hash)" do
|
34
|
+
it "changes top level string keys to symbols" do
|
35
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
36
|
+
"test" => 1,
|
37
|
+
)
|
38
|
+
|
39
|
+
expect(symbolized_hash).to eq(
|
40
|
+
test: 1,
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "changes lower level string keys to symbols" do
|
45
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
46
|
+
test: {
|
47
|
+
"test" => 1,
|
48
|
+
},
|
49
|
+
)
|
50
|
+
|
51
|
+
expect(symbolized_hash).to eq(
|
52
|
+
test: {
|
53
|
+
test: 1,
|
54
|
+
},
|
55
|
+
)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "leaves existing symbol keys alone" do
|
59
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
60
|
+
test: 1,
|
61
|
+
)
|
62
|
+
|
63
|
+
expect(symbolized_hash).to eq(
|
64
|
+
test: 1,
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "leaves other object keys alone" do
|
69
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
70
|
+
1 => "test",
|
71
|
+
)
|
72
|
+
|
73
|
+
expect(symbolized_hash).to eq(
|
74
|
+
1 => "test",
|
75
|
+
)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
metadata
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: async_experiments
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elliot Crosby-McCullough
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-06-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: hashdiff
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3.4'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3.4'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '11.2'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '11.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.10'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.10'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: gem_publisher
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 1.5.0
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.5.0
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: timecop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sidekiq
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description:
|
126
|
+
email:
|
127
|
+
- elliot.cm@gmail.com
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- LICENCE.txt
|
133
|
+
- README.md
|
134
|
+
- lib/async_experiments.rb
|
135
|
+
- lib/async_experiments/candidate_worker.rb
|
136
|
+
- lib/async_experiments/experiment_control.rb
|
137
|
+
- lib/async_experiments/experiment_error_worker.rb
|
138
|
+
- lib/async_experiments/experiment_result.rb
|
139
|
+
- lib/async_experiments/experiment_result_worker.rb
|
140
|
+
- lib/async_experiments/util.rb
|
141
|
+
- lib/async_experiments/version.rb
|
142
|
+
- spec/async_experiments_spec.rb
|
143
|
+
- spec/candidate_worker_spec.rb
|
144
|
+
- spec/examples.txt
|
145
|
+
- spec/experiment_control_spec.rb
|
146
|
+
- spec/experiment_error_worker_spec.rb
|
147
|
+
- spec/experiment_result_spec.rb
|
148
|
+
- spec/experiment_result_worker_spec.rb
|
149
|
+
- spec/spec_helper.rb
|
150
|
+
- spec/util_spec.rb
|
151
|
+
homepage: http://github.com/alphagov/async_experiments
|
152
|
+
licenses:
|
153
|
+
- MIT
|
154
|
+
metadata: {}
|
155
|
+
post_install_message:
|
156
|
+
rdoc_options: []
|
157
|
+
require_paths:
|
158
|
+
- lib
|
159
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">="
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: '0'
|
169
|
+
requirements: []
|
170
|
+
rubyforge_project:
|
171
|
+
rubygems_version: 2.5.1
|
172
|
+
signing_key:
|
173
|
+
specification_version: 4
|
174
|
+
summary: An asynchronous experiment framework.
|
175
|
+
test_files:
|
176
|
+
- spec/experiment_result_spec.rb
|
177
|
+
- spec/spec_helper.rb
|
178
|
+
- spec/experiment_error_worker_spec.rb
|
179
|
+
- spec/experiment_result_worker_spec.rb
|
180
|
+
- spec/async_experiments_spec.rb
|
181
|
+
- spec/util_spec.rb
|
182
|
+
- spec/experiment_control_spec.rb
|
183
|
+
- spec/candidate_worker_spec.rb
|
184
|
+
- spec/examples.txt
|