async_experiments 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENCE.txt +21 -0
- data/README.md +127 -0
- data/lib/async_experiments/candidate_worker.rb +29 -0
- data/lib/async_experiments/experiment_control.rb +27 -0
- data/lib/async_experiments/experiment_error_worker.rb +14 -0
- data/lib/async_experiments/experiment_result.rb +90 -0
- data/lib/async_experiments/experiment_result_worker.rb +36 -0
- data/lib/async_experiments/util.rb +20 -0
- data/lib/async_experiments/version.rb +3 -0
- data/lib/async_experiments.rb +51 -0
- data/spec/async_experiments_spec.rb +49 -0
- data/spec/candidate_worker_spec.rb +96 -0
- data/spec/examples.txt +37 -0
- data/spec/experiment_control_spec.rb +60 -0
- data/spec/experiment_error_worker_spec.rb +36 -0
- data/spec/experiment_result_spec.rb +136 -0
- data/spec/experiment_result_worker_spec.rb +106 -0
- data/spec/spec_helper.rb +101 -0
- data/spec/util_spec.rb +78 -0
- metadata +184 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 73d448d952cbbee2567c807bafd327fead90ac38
|
4
|
+
data.tar.gz: 47f6da2a76c6f75c42ea8904c72421164bc77c3c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c3d83b96b7143a1594c36d3d29b3ba77e6dd65409a35201dae68059cfdc8e2ffe4e03e28b6805d58251a82340984daee93eaf1ec40c03c820caaaf0e85b28ca2
|
7
|
+
data.tar.gz: a8932415d200081e7a6176175cbd29372a5e91463cdc040e5d72a5424939af9e10176ef149efa891868af3e4c84cc009ffe06fd5bf99f5ebdf55ee24b47c0100
|
data/LICENCE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Crown Copyright (Government Digital Service)
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
# Asynchronous Experiments
|
2
|
+
|
3
|
+
Similar to GitHub Scientist, but uses Sidekiq (and its Redis connection pool)
|
4
|
+
to run control and candidate branches of experiments in parallel, storing the
|
5
|
+
output for later review.
|
6
|
+
|
7
|
+
Provides helpers to assist with rendering the output from the comparison.
|
8
|
+
|
9
|
+
## IMPORTANT NOTE ABOUT SIDEKIQ
|
10
|
+
|
11
|
+
This gem expects Sidekiq to be included, but does not list it as a gem dependency.
|
12
|
+
|
13
|
+
This is because GOV.UK uses `govuk_sidekiq`, a gem which automatically configures
|
14
|
+
our apps with some standard Sidekiq configurations. We haven't added that to the
|
15
|
+
gemspec as other organisations may want to use other methods to configure Sidekiq.
|
16
|
+
|
17
|
+
The gem also assumes access to statsd for reporting purposes.
|
18
|
+
|
19
|
+
## Technical documentation
|
20
|
+
|
21
|
+
Example usage:
|
22
|
+
```
|
23
|
+
require "async_experiments/experiment_control"
|
24
|
+
|
25
|
+
class ContentItemsController < ApplicationController
|
26
|
+
include AsyncExperiments::ExperimentControl
|
27
|
+
|
28
|
+
def linkables
|
29
|
+
candidate = {
|
30
|
+
worker: LinkablesCandidate,
|
31
|
+
args: [
|
32
|
+
query_params.fetch(:document_type),
|
33
|
+
],
|
34
|
+
}
|
35
|
+
|
36
|
+
presented = experiment_control(:linkables, candidate: candidate) {
|
37
|
+
Queries::GetContentCollection.new(
|
38
|
+
document_type: query_params.fetch(:document_type),
|
39
|
+
fields: %w(
|
40
|
+
title
|
41
|
+
content_id
|
42
|
+
publication_state
|
43
|
+
base_path
|
44
|
+
internal_name
|
45
|
+
),
|
46
|
+
pagination: NullPagination.new
|
47
|
+
).call
|
48
|
+
}
|
49
|
+
|
50
|
+
render json: presented
|
51
|
+
end
|
52
|
+
end
|
53
|
+
```
|
54
|
+
|
55
|
+
```
|
56
|
+
require "async_experiments/candidate_worker"
|
57
|
+
|
58
|
+
class LinkablesCandidate < AsyncExperiments::CandidateWorker
|
59
|
+
def perform(document_type, experiment)
|
60
|
+
experiment_candidate(experiment) do
|
61
|
+
Queries::GetLinkables.new(
|
62
|
+
document_type: document_type,
|
63
|
+
).call
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
```
|
68
|
+
|
69
|
+
```
|
70
|
+
class DebugController < ApplicationController
|
71
|
+
skip_before_action :require_signin_permission!
|
72
|
+
before_action :validate_experiment_name, only: [:experiment]
|
73
|
+
|
74
|
+
def experiment
|
75
|
+
@mismatched_responses = AsyncExperiments.get_experiment_data(params[:experiment])
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def validate_experiment_name
|
81
|
+
raise "Experiment names don't contain `:`" if params[:experiment].include?(":")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
```
|
85
|
+
|
86
|
+
```
|
87
|
+
<ul>
|
88
|
+
<% @mismatched_responses.each_with_index do |mismatch, i| %>
|
89
|
+
<li>
|
90
|
+
<ul>
|
91
|
+
<li><a href="#missing-#{i}">Missing</a></li>
|
92
|
+
<li><a href="#extra-#{i}">Extra</a></li>
|
93
|
+
<li><a href="#changed-#{i}">Changed</a></li>
|
94
|
+
</ul>
|
95
|
+
|
96
|
+
<h3 id="missing-#{i}">Missing from candidate</h3>
|
97
|
+
<% mismatch[:missing].each do |entry| %>
|
98
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
99
|
+
<% end %>
|
100
|
+
|
101
|
+
<h3 id="extra-#{i}">Extra in candidate</h3>
|
102
|
+
<% mismatch[:extra].each do |entry| %>
|
103
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
104
|
+
<% end %>
|
105
|
+
|
106
|
+
<h3 id="changed-#{i}">Changed in candidate</h3>
|
107
|
+
<% mismatch[:changed].each do |entry| %>
|
108
|
+
<pre><%= PP.pp(entry, "") %></pre>
|
109
|
+
<% end %>
|
110
|
+
</li>
|
111
|
+
<% end %>
|
112
|
+
</ul>
|
113
|
+
```
|
114
|
+
|
115
|
+
```
|
116
|
+
statsd_client = Statsd.new("localhost")
|
117
|
+
statsd_client.namespace = "govuk.app.publishing-api"
|
118
|
+
AsyncExperiments.statsd = statsd_client
|
119
|
+
```
|
120
|
+
|
121
|
+
## Licence
|
122
|
+
|
123
|
+
[MIT License](LICENCE)
|
124
|
+
|
125
|
+
## Versioning policy
|
126
|
+
|
127
|
+
See https://github.com/alphagov/styleguides/blob/master/rubygems.md#versioning
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "async_experiments/experiment_result_worker"
|
2
|
+
require "async_experiments/experiment_error_worker"
|
3
|
+
|
4
|
+
module AsyncExperiments
|
5
|
+
class CandidateWorker
|
6
|
+
include Sidekiq::Worker
|
7
|
+
|
8
|
+
sidekiq_options queue: :experiments
|
9
|
+
|
10
|
+
def experiment_candidate(experiment_config)
|
11
|
+
experiment = experiment_config.symbolize_keys
|
12
|
+
|
13
|
+
start_time = Time.now
|
14
|
+
run_output = yield
|
15
|
+
duration = (Time.now - start_time).to_f
|
16
|
+
ExperimentResultWorker.perform_async(experiment[:name], experiment[:id], run_output, duration, :candidate)
|
17
|
+
|
18
|
+
run_output
|
19
|
+
rescue StandardError => exception
|
20
|
+
if ENV["RAISE_EXPERIMENT_ERRORS"]
|
21
|
+
raise exception
|
22
|
+
else
|
23
|
+
backtrace = exception.backtrace
|
24
|
+
backtrace.unshift(exception.inspect)
|
25
|
+
ExperimentErrorWorker.perform_async(experiment[:name], backtrace.join("\n"))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "async_experiments/experiment_result_worker"
|
2
|
+
|
3
|
+
module AsyncExperiments
|
4
|
+
module ExperimentControl
|
5
|
+
def experiment_control(name, candidate:)
|
6
|
+
start_time = Time.now
|
7
|
+
run_output = yield
|
8
|
+
duration = (Time.now - start_time).to_f
|
9
|
+
|
10
|
+
id = SecureRandom.uuid
|
11
|
+
|
12
|
+
if run_output.class == Enumerator
|
13
|
+
run_output = run_output.to_a
|
14
|
+
end
|
15
|
+
|
16
|
+
ExperimentResultWorker.perform_async(name, id, run_output, duration, :control)
|
17
|
+
|
18
|
+
candidate_worker = candidate.fetch(:worker)
|
19
|
+
candidate_worker.perform_async(*candidate.fetch(:args),
|
20
|
+
name: name,
|
21
|
+
id: id,
|
22
|
+
)
|
23
|
+
|
24
|
+
run_output
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module AsyncExperiments
|
2
|
+
class ExperimentErrorWorker
|
3
|
+
include Sidekiq::Worker
|
4
|
+
|
5
|
+
sidekiq_options queue: :experiments
|
6
|
+
|
7
|
+
def perform(experiment_name, exception_string)
|
8
|
+
Sidekiq.redis do |redis|
|
9
|
+
AsyncExperiments.statsd.increment("experiments.#{experiment_name}.exceptions")
|
10
|
+
redis.rpush("experiments:#{experiment_name}:exceptions", exception_string)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require "json"
|
2
|
+
require "hashdiff"
|
3
|
+
require "async_experiments/util"
|
4
|
+
|
5
|
+
module AsyncExperiments
|
6
|
+
class ExperimentResult
|
7
|
+
def initialize(name, id, type, redis, statsd, run_output = nil, duration = nil)
|
8
|
+
@name = name
|
9
|
+
@key = "#{name}:#{id}"
|
10
|
+
@redis = redis
|
11
|
+
@statsd = statsd
|
12
|
+
@type = type
|
13
|
+
@run_output = run_output
|
14
|
+
@duration = duration
|
15
|
+
|
16
|
+
if Util.blank?(run_output) || Util.blank?(duration)
|
17
|
+
redis_data = data_from_redis
|
18
|
+
|
19
|
+
if redis_data
|
20
|
+
@run_output ||= redis_data.fetch(:run_output)
|
21
|
+
@duration ||= redis_data.fetch(:duration)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
attr_reader :key, :run_output, :duration
|
27
|
+
|
28
|
+
def store_run_output
|
29
|
+
redis.set("experiments:#{key}:#{type}", {
|
30
|
+
run_output: run_output,
|
31
|
+
duration: duration,
|
32
|
+
}.to_json)
|
33
|
+
end
|
34
|
+
|
35
|
+
def process_run_output(candidate)
|
36
|
+
variation = HashDiff.diff(sort(self.run_output), sort(candidate.run_output))
|
37
|
+
report_data(variation, candidate)
|
38
|
+
redis.del("experiments:#{key}:candidate")
|
39
|
+
end
|
40
|
+
|
41
|
+
def control?
|
42
|
+
type == :control
|
43
|
+
end
|
44
|
+
|
45
|
+
def candidate?
|
46
|
+
type == :candidate
|
47
|
+
end
|
48
|
+
|
49
|
+
def available?
|
50
|
+
Util.present?(run_output) && Util.present?(duration)
|
51
|
+
end
|
52
|
+
|
53
|
+
protected
|
54
|
+
|
55
|
+
attr_reader :redis, :statsd, :type, :name
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def data_from_redis
|
60
|
+
redis_data = redis.get("experiments:#{key}:#{type}")
|
61
|
+
|
62
|
+
if Util.present?(redis_data)
|
63
|
+
Util.deep_symbolize_keys(JSON.parse(redis_data))
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def report_data(variation, candidate)
|
68
|
+
statsd.timing("experiments.#{name}.control", self.duration)
|
69
|
+
statsd.timing("experiments.#{name}.candidate", candidate.duration)
|
70
|
+
|
71
|
+
if variation != []
|
72
|
+
statsd.increment("experiments.#{name}.mismatches")
|
73
|
+
redis.rpush("experiments:#{name}:mismatches", JSON.dump(variation))
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def sort(object)
|
78
|
+
case object
|
79
|
+
when Array
|
80
|
+
object.sort_by(&:object_id)
|
81
|
+
when Hash
|
82
|
+
object.each_with_object({}) { |(key, value), hash|
|
83
|
+
hash[key] = sort(value)
|
84
|
+
}
|
85
|
+
else
|
86
|
+
object
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "async_experiments/experiment_result"
|
2
|
+
|
3
|
+
module AsyncExperiments
|
4
|
+
class ExperimentResultWorker
|
5
|
+
include Sidekiq::Worker
|
6
|
+
|
7
|
+
sidekiq_options queue: :experiments
|
8
|
+
|
9
|
+
LOCK_TIMEOUT = 60
|
10
|
+
|
11
|
+
def perform(name, id, run_output, duration, type)
|
12
|
+
type = type.to_sym
|
13
|
+
|
14
|
+
Sidekiq.redis do |redis|
|
15
|
+
this_branch = ExperimentResult.new(name, id, type, redis, statsd, run_output, duration)
|
16
|
+
|
17
|
+
if this_branch.control?
|
18
|
+
candidate = ExperimentResult.new(name, id, :candidate, redis, statsd)
|
19
|
+
if candidate.available?
|
20
|
+
this_branch.process_run_output(candidate)
|
21
|
+
else
|
22
|
+
self.class.perform_in(5, name, id, run_output, duration, type)
|
23
|
+
end
|
24
|
+
elsif this_branch.candidate?
|
25
|
+
this_branch.store_run_output
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def statsd
|
33
|
+
AsyncExperiments.statsd
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module AsyncExperiments
|
2
|
+
module Util
|
3
|
+
def self.present?(object)
|
4
|
+
!self.blank?(object)
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.blank?(object)
|
8
|
+
object.nil? || (object.respond_to?(:empty?) && object.empty?)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.deep_symbolize_keys(hash)
|
12
|
+
return hash unless hash.is_a?(Hash)
|
13
|
+
|
14
|
+
hash.each_with_object({}) do |(key, value), new_hash|
|
15
|
+
key = key.respond_to?(:to_sym) ? key.to_sym : key
|
16
|
+
new_hash[key] = self.deep_symbolize_keys(value)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require "json"
|
2
|
+
require "securerandom"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
module AsyncExperiments
|
6
|
+
def self.statsd
|
7
|
+
@statsd
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.statsd=(statsd)
|
11
|
+
@statsd = statsd
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.get_experiment_data(experiment_name)
|
15
|
+
mismatched_responses = Sidekiq.redis { |redis|
|
16
|
+
redis.lrange("experiments:#{experiment_name}:mismatches", 0, -1)
|
17
|
+
}
|
18
|
+
|
19
|
+
mismatched_responses.map { |json|
|
20
|
+
parsed = JSON.parse(json)
|
21
|
+
|
22
|
+
missing, other = parsed.partition {|(operator, _, _)|
|
23
|
+
operator == "-"
|
24
|
+
}
|
25
|
+
|
26
|
+
extra, changed = other.partition {|(operator, _, _)|
|
27
|
+
operator == "+"
|
28
|
+
}
|
29
|
+
|
30
|
+
missing_entries, extra_entries = self.fix_ordering_issues(
|
31
|
+
missing.map(&:last),
|
32
|
+
extra.map(&:last),
|
33
|
+
)
|
34
|
+
|
35
|
+
{
|
36
|
+
missing: missing_entries,
|
37
|
+
extra: extra_entries,
|
38
|
+
changed: changed.map(&:last),
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.fix_ordering_issues(missing_entries, extra_entries)
|
44
|
+
duplicate_entries = missing_entries & extra_entries
|
45
|
+
|
46
|
+
missing_entries = missing_entries.reject { |entry| duplicate_entries.include?(entry) }
|
47
|
+
extra_entries = extra_entries.reject { |entry| duplicate_entries.include?(entry) }
|
48
|
+
|
49
|
+
[missing_entries, extra_entries]
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "json"
|
3
|
+
require "async_experiments"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments do
|
6
|
+
describe ".get_experiment_data(experiment_name)" do
|
7
|
+
let(:name) { "some_experiment" }
|
8
|
+
|
9
|
+
let(:experiment_results) {
|
10
|
+
[
|
11
|
+
JSON.dump([
|
12
|
+
["-", "[1]", {same_key: 1, different_key: 2}],
|
13
|
+
["+", "[2]", {same_key: 1, different_key: 3}],
|
14
|
+
["+", "[3]", "Extra element"],
|
15
|
+
["-", "[4]", "Missing element"],
|
16
|
+
["~", "[5]", "Changed element"],
|
17
|
+
["-", "[3]", {moved_complex_object: 1}],
|
18
|
+
["+", "[6]", {moved_complex_object: 1}],
|
19
|
+
])
|
20
|
+
]
|
21
|
+
}
|
22
|
+
|
23
|
+
let(:redis) { double(:redis) }
|
24
|
+
|
25
|
+
before do
|
26
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
27
|
+
allow(redis).to receive(:lrange).with("experiments:#{name}:mismatches", 0, -1)
|
28
|
+
.and_return(experiment_results)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "partitions and resorts experiment results for useful output" do
|
32
|
+
results = described_class.get_experiment_data(name)
|
33
|
+
|
34
|
+
expect(results).to eq([
|
35
|
+
missing: [
|
36
|
+
{"same_key" => 1, "different_key" => 2},
|
37
|
+
"Missing element",
|
38
|
+
],
|
39
|
+
extra: [
|
40
|
+
{"same_key" => 1, "different_key" => 3},
|
41
|
+
"Extra element",
|
42
|
+
],
|
43
|
+
changed: [
|
44
|
+
"Changed element",
|
45
|
+
],
|
46
|
+
])
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/candidate_worker"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::CandidateWorker do
|
6
|
+
let(:name) { "some_experiment" }
|
7
|
+
let(:id) { SecureRandom.uuid }
|
8
|
+
|
9
|
+
let(:run_output) { [{some: "output"}] }
|
10
|
+
|
11
|
+
class TestWorker < AsyncExperiments::CandidateWorker
|
12
|
+
def perform(run_output, experiment_config)
|
13
|
+
experiment_candidate(experiment_config) do
|
14
|
+
Timecop.travel(Time.now + 1.5)
|
15
|
+
run_output
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
subject { TestWorker.new }
|
21
|
+
|
22
|
+
it "uses the 'experiments' queue" do
|
23
|
+
Sidekiq::Testing.fake! do
|
24
|
+
TestWorker.perform_async
|
25
|
+
|
26
|
+
expect(TestWorker.jobs.size).to eq(1)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
it "returns the control run output" do
|
32
|
+
output = subject.perform(run_output,
|
33
|
+
name: name,
|
34
|
+
id: id,
|
35
|
+
)
|
36
|
+
|
37
|
+
expect(output).to eq(run_output)
|
38
|
+
end
|
39
|
+
|
40
|
+
describe "#experiment_candidate(experiment_config)" do
|
41
|
+
it "triggers an ExperimentResultWorker with the candidate output and duration" do
|
42
|
+
expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
|
43
|
+
.with(name, id, run_output, instance_of(Float), :candidate)
|
44
|
+
|
45
|
+
subject.perform(run_output,
|
46
|
+
name: name,
|
47
|
+
id: id,
|
48
|
+
)
|
49
|
+
end
|
50
|
+
|
51
|
+
context "when experiment errors are being raised" do
|
52
|
+
around do |example|
|
53
|
+
setting = ENV["RAISE_EXPERIMENT_ERRORS"]
|
54
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
|
55
|
+
|
56
|
+
example.run
|
57
|
+
|
58
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = setting
|
59
|
+
end
|
60
|
+
|
61
|
+
it "re-raises experiment errors" do
|
62
|
+
allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
|
63
|
+
|
64
|
+
expect {
|
65
|
+
subject.perform(run_output,
|
66
|
+
"name" => name,
|
67
|
+
"id" => id,
|
68
|
+
)
|
69
|
+
}.to raise_error(Exception, "Test exception")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
context "when experiment errors are being quietly reported" do
|
74
|
+
around do |example|
|
75
|
+
setting = ENV["RAISE_EXPERIMENT_ERRORS"]
|
76
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = nil
|
77
|
+
|
78
|
+
example.run
|
79
|
+
|
80
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = setting
|
81
|
+
end
|
82
|
+
|
83
|
+
it "re-raises experiment errors" do
|
84
|
+
allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
|
85
|
+
|
86
|
+
expect(AsyncExperiments::ExperimentErrorWorker).to receive(:perform_async)
|
87
|
+
.with(name, instance_of(String))
|
88
|
+
|
89
|
+
subject.perform(run_output,
|
90
|
+
name: name,
|
91
|
+
id: id,
|
92
|
+
)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
data/spec/examples.txt
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
example_id | status | run_time |
|
2
|
+
------------------------------------------------ | ------ | --------------- |
|
3
|
+
./spec/async_experiments_spec.rb[1:1:1] | passed | 0.00062 seconds |
|
4
|
+
./spec/candidate_worker_spec.rb[1:1] | passed | 0.00023 seconds |
|
5
|
+
./spec/candidate_worker_spec.rb[1:2] | passed | 0.00053 seconds |
|
6
|
+
./spec/candidate_worker_spec.rb[1:3:1] | passed | 0.00069 seconds |
|
7
|
+
./spec/candidate_worker_spec.rb[1:3:2:1] | passed | 0.00209 seconds |
|
8
|
+
./spec/candidate_worker_spec.rb[1:3:3:1] | passed | 0.0008 seconds |
|
9
|
+
./spec/experiment_control_spec.rb[1:1:1] | passed | 0.00089 seconds |
|
10
|
+
./spec/experiment_control_spec.rb[1:1:2] | passed | 0.00117 seconds |
|
11
|
+
./spec/experiment_control_spec.rb[1:1:3] | passed | 0.00061 seconds |
|
12
|
+
./spec/experiment_error_worker_spec.rb[1:1] | passed | 0.00266 seconds |
|
13
|
+
./spec/experiment_error_worker_spec.rb[1:2] | passed | 0.00056 seconds |
|
14
|
+
./spec/experiment_error_worker_spec.rb[1:3] | passed | 0.0077 seconds |
|
15
|
+
./spec/experiment_result_spec.rb[1:1:1] | passed | 0.00042 seconds |
|
16
|
+
./spec/experiment_result_spec.rb[1:2:1] | passed | 0.00043 seconds |
|
17
|
+
./spec/experiment_result_spec.rb[1:3:1] | passed | 0.00071 seconds |
|
18
|
+
./spec/experiment_result_spec.rb[1:3:2] | passed | 0.00045 seconds |
|
19
|
+
./spec/experiment_result_spec.rb[1:3:3:1] | passed | 0.00055 seconds |
|
20
|
+
./spec/experiment_result_spec.rb[1:3:3:2] | passed | 0.00048 seconds |
|
21
|
+
./spec/experiment_result_spec.rb[1:3:4:1] | passed | 0.00043 seconds |
|
22
|
+
./spec/experiment_result_spec.rb[1:3:4:2] | passed | 0.00058 seconds |
|
23
|
+
./spec/experiment_result_spec.rb[1:4:1:1] | passed | 0.00037 seconds |
|
24
|
+
./spec/experiment_result_spec.rb[1:4:2:1:1] | passed | 0.00095 seconds |
|
25
|
+
./spec/experiment_result_spec.rb[1:4:2:1:2] | passed | 0.00055 seconds |
|
26
|
+
./spec/experiment_result_spec.rb[1:4:2:2:1] | passed | 0.0005 seconds |
|
27
|
+
./spec/experiment_result_worker_spec.rb[1:1] | passed | 0.00048 seconds |
|
28
|
+
./spec/experiment_result_worker_spec.rb[1:2:1] | passed | 0.00175 seconds |
|
29
|
+
./spec/experiment_result_worker_spec.rb[1:3:1:1] | passed | 0.0013 seconds |
|
30
|
+
./spec/experiment_result_worker_spec.rb[1:3:2:1] | passed | 0.00235 seconds |
|
31
|
+
./spec/experiment_result_worker_spec.rb[1:3:2:2] | passed | 0.0025 seconds |
|
32
|
+
./spec/util_spec.rb[1:1:1] | passed | 0.00021 seconds |
|
33
|
+
./spec/util_spec.rb[1:2:1] | passed | 0.00016 seconds |
|
34
|
+
./spec/util_spec.rb[1:3:1] | passed | 0.00014 seconds |
|
35
|
+
./spec/util_spec.rb[1:3:2] | passed | 0.00016 seconds |
|
36
|
+
./spec/util_spec.rb[1:3:3] | passed | 0.00012 seconds |
|
37
|
+
./spec/util_spec.rb[1:3:4] | passed | 0.00011 seconds |
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/experiment_control"
|
3
|
+
require "async_experiments/experiment_result_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::ExperimentControl do
|
6
|
+
let(:name) { :some_experiment }
|
7
|
+
let(:id) { double(:id) }
|
8
|
+
|
9
|
+
let(:candidate_args) { [1, 2, 3] }
|
10
|
+
let(:candidate_worker) { double(:candidate_worker, perform_async: nil) }
|
11
|
+
let(:candidate_config) {
|
12
|
+
{
|
13
|
+
worker: candidate_worker,
|
14
|
+
args: candidate_args,
|
15
|
+
}
|
16
|
+
}
|
17
|
+
|
18
|
+
let(:run_output) { [{some: "output"}] }
|
19
|
+
|
20
|
+
class TestClass
|
21
|
+
include AsyncExperiments::ExperimentControl
|
22
|
+
|
23
|
+
def call(name, run_output, candidate_config)
|
24
|
+
experiment_control(name, candidate: candidate_config) do
|
25
|
+
Timecop.travel(Time.now + 1.5)
|
26
|
+
run_output
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
subject { TestClass.new }
|
32
|
+
|
33
|
+
before do
|
34
|
+
allow(SecureRandom).to receive(:uuid).and_return(id)
|
35
|
+
end
|
36
|
+
|
37
|
+
describe "#experiment_control(name, candidate: candidate_config)" do
|
38
|
+
it "triggers an ExperimentResultWorker with the control output and duration" do
|
39
|
+
expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
|
40
|
+
.with(name, id, run_output, instance_of(Float), :control)
|
41
|
+
|
42
|
+
subject.call(name, run_output, candidate_config)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "triggers the candidate worker with its arguments and the experiment config" do
|
46
|
+
expect(candidate_worker).to receive(:perform_async)
|
47
|
+
.with(*candidate_args,
|
48
|
+
name: name,
|
49
|
+
id: id,
|
50
|
+
)
|
51
|
+
|
52
|
+
subject.call(name, run_output, candidate_config)
|
53
|
+
end
|
54
|
+
|
55
|
+
it "returns the control run output" do
|
56
|
+
output = subject.call(name, run_output, candidate_config)
|
57
|
+
expect(output).to eq(run_output)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments"
|
3
|
+
require "async_experiments/experiment_error_worker"
|
4
|
+
|
5
|
+
RSpec.describe AsyncExperiments::ExperimentErrorWorker do
|
6
|
+
let(:name) { "some_experiment" }
|
7
|
+
let(:error) { "Something went wrong" }
|
8
|
+
|
9
|
+
let(:statsd) { double(:statsd, increment: nil) }
|
10
|
+
let(:redis) { double(:redis, rpush: nil) }
|
11
|
+
|
12
|
+
subject { described_class.new }
|
13
|
+
|
14
|
+
before do
|
15
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
16
|
+
AsyncExperiments.statsd = statsd
|
17
|
+
end
|
18
|
+
|
19
|
+
it "uses the 'experiments' queue" do
|
20
|
+
Sidekiq::Testing.fake! do
|
21
|
+
described_class.perform_async
|
22
|
+
|
23
|
+
expect(described_class.jobs.size).to eq(1)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it "increments the statsd error count for the experiment" do
|
28
|
+
expect(statsd).to receive(:increment).with("experiments.#{name}.exceptions")
|
29
|
+
subject.perform(name, error)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "stores the exception for later reporting" do
|
33
|
+
expect(redis).to receive(:rpush).with("experiments:#{name}:exceptions", error)
|
34
|
+
subject.perform(name, error)
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/experiment_result"
|
3
|
+
|
4
|
+
RSpec.describe AsyncExperiments::ExperimentResult do
|
5
|
+
let(:id) { SecureRandom.uuid }
|
6
|
+
let(:name) { :test_experiment }
|
7
|
+
|
8
|
+
let(:control_run_output) { "control output" }
|
9
|
+
let(:control_duration) { 10.0 }
|
10
|
+
|
11
|
+
let(:candidate_run_output) { "candidate output" }
|
12
|
+
let(:candidate_duration) { 5.0 }
|
13
|
+
|
14
|
+
let(:redis) { double(:redis, del: nil) }
|
15
|
+
let(:statsd) { double(:statsd, timing: nil, increment: nil) }
|
16
|
+
|
17
|
+
let(:control) { described_class.new(name, id, :control, redis, statsd, control_run_output, control_duration) }
|
18
|
+
let(:candidate) { described_class.new(name, id, :candidate, redis, statsd, candidate_run_output, candidate_duration) }
|
19
|
+
|
20
|
+
describe "#key" do
|
21
|
+
it "builds a key from the name and ID" do
|
22
|
+
expect(control.key).to eq("test_experiment:#{id}")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "#store_run_output" do
|
27
|
+
it "stores the branch's run output and duration" do
|
28
|
+
expect(redis).to receive(:set)
|
29
|
+
.with("experiments:#{name}:#{id}:candidate", {
|
30
|
+
run_output: candidate_run_output,
|
31
|
+
duration: candidate_duration,
|
32
|
+
}.to_json)
|
33
|
+
|
34
|
+
candidate.store_run_output
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "control#process_run_output(candidate)" do
|
39
|
+
before do
|
40
|
+
allow(redis).to receive(:rpush)
|
41
|
+
end
|
42
|
+
|
43
|
+
it "reports the control and candidate durations to statsd" do
|
44
|
+
expect(statsd).to receive(:timing)
|
45
|
+
.with("experiments.#{name}.control", control_duration)
|
46
|
+
|
47
|
+
expect(statsd).to receive(:timing)
|
48
|
+
.with("experiments.#{name}.candidate", candidate_duration)
|
49
|
+
|
50
|
+
control.process_run_output(candidate)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "deletes the candidate data from redis" do
|
54
|
+
expect(redis).to receive(:del).with("experiments:#{name}:#{id}:candidate")
|
55
|
+
|
56
|
+
control.process_run_output(candidate)
|
57
|
+
end
|
58
|
+
|
59
|
+
context "if there's variation between the outputs" do
|
60
|
+
it "increments the mismatch count in statsd" do
|
61
|
+
expect(statsd).to receive(:increment)
|
62
|
+
.with("experiments.#{name}.mismatches")
|
63
|
+
|
64
|
+
control.process_run_output(candidate)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "logs the mismatch to redis" do
|
68
|
+
expect(redis).to receive(:rpush).with(
|
69
|
+
"experiments:#{name}:mismatches",
|
70
|
+
[["~", "", "control output", "candidate output"]].to_json,
|
71
|
+
)
|
72
|
+
|
73
|
+
control.process_run_output(candidate)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context "if there's no variation" do
|
78
|
+
let(:candidate_run_output) { control_run_output }
|
79
|
+
|
80
|
+
it "does not increment the mismatch count" do
|
81
|
+
expect(statsd).not_to receive(:increment)
|
82
|
+
control.process_run_output(candidate)
|
83
|
+
end
|
84
|
+
|
85
|
+
it "does not log the mismatch to redis" do
|
86
|
+
expect(redis).not_to receive(:rpush)
|
87
|
+
control.process_run_output(candidate)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe ".new" do
|
93
|
+
let(:type) { :candidate }
|
94
|
+
|
95
|
+
context "if duration and output are provided" do
|
96
|
+
it "uses those" do
|
97
|
+
candidate = described_class.new(name, id, type, redis, statsd, "arbitrary output", 1.23)
|
98
|
+
expect(candidate.run_output).to eq("arbitrary output")
|
99
|
+
expect(candidate.duration).to eq(1.23)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
context "if duration and output are not provided" do
|
104
|
+
context "and redis has the data" do
|
105
|
+
before do
|
106
|
+
allow(redis).to receive(:get).with("experiments:#{name}:#{id}:#{type}").and_return({
|
107
|
+
run_output: candidate_run_output,
|
108
|
+
duration: candidate_duration,
|
109
|
+
}.to_json)
|
110
|
+
end
|
111
|
+
|
112
|
+
it "uses the redis data" do
|
113
|
+
candidate = described_class.new(name, id, type, redis, statsd)
|
114
|
+
expect(candidate.run_output).to eq(candidate_run_output)
|
115
|
+
expect(candidate.duration).to eq(candidate_duration)
|
116
|
+
end
|
117
|
+
|
118
|
+
it "is considered available" do
|
119
|
+
candidate = described_class.new(name, id, type, redis, statsd)
|
120
|
+
expect(candidate.available?).to eq(true)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
context "but redis does not have the data" do
|
125
|
+
before do
|
126
|
+
allow(redis).to receive(:get).and_return("")
|
127
|
+
end
|
128
|
+
|
129
|
+
it "is considered unavailable" do
|
130
|
+
missing_candidate = described_class.new(name, id, type, redis, statsd)
|
131
|
+
expect(missing_candidate.available?).to eq(false)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments"
|
3
|
+
require "async_experiments/experiment_result"
|
4
|
+
require "async_experiments/experiment_result_worker"
|
5
|
+
|
6
|
+
RSpec.describe AsyncExperiments::ExperimentResultWorker do
|
7
|
+
let(:name) { "some_experiment" }
|
8
|
+
let(:id) { SecureRandom.uuid }
|
9
|
+
|
10
|
+
let(:statsd) { double(:statsd) }
|
11
|
+
let(:redis) { double(:redis) }
|
12
|
+
let(:redis_key) { double(:redis_key) }
|
13
|
+
|
14
|
+
let(:control_output) { "control output" }
|
15
|
+
let(:control_duration) { 10.0 }
|
16
|
+
|
17
|
+
let(:candidate_output) { "candidate output" }
|
18
|
+
let(:candidate_duration) { 5.0 }
|
19
|
+
|
20
|
+
let(:control) {
|
21
|
+
double(:control,
|
22
|
+
control?: true,
|
23
|
+
candidate?: false,
|
24
|
+
key: redis_key,
|
25
|
+
)
|
26
|
+
}
|
27
|
+
let(:candidate) {
|
28
|
+
double(:candidate,
|
29
|
+
candidate?: true,
|
30
|
+
control?: false,
|
31
|
+
key: redis_key,
|
32
|
+
)
|
33
|
+
}
|
34
|
+
|
35
|
+
subject { described_class.new }
|
36
|
+
|
37
|
+
before do
|
38
|
+
allow(Sidekiq).to receive(:redis).and_yield(redis)
|
39
|
+
AsyncExperiments.statsd = statsd
|
40
|
+
end
|
41
|
+
|
42
|
+
it "uses the 'experiments' queue" do
|
43
|
+
Sidekiq::Testing.fake! do
|
44
|
+
described_class.perform_async
|
45
|
+
|
46
|
+
expect(described_class.jobs.size).to eq(1)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "if we're the candidate" do
|
51
|
+
let(:type) { "candidate" }
|
52
|
+
|
53
|
+
it "stores the run output and duration" do
|
54
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
55
|
+
.with(name, id, type.to_sym, redis, statsd, candidate_output, candidate_duration)
|
56
|
+
.and_return(candidate)
|
57
|
+
|
58
|
+
expect(candidate).to receive(:store_run_output)
|
59
|
+
|
60
|
+
subject.perform(name, id, candidate_output, candidate_duration, type)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
context "if we're the control" do
|
65
|
+
let(:type) { "control" }
|
66
|
+
|
67
|
+
before do
|
68
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
69
|
+
.with(name, id, :control, redis, statsd, control_output, control_duration)
|
70
|
+
.and_return(control)
|
71
|
+
|
72
|
+
expect(AsyncExperiments::ExperimentResult).to receive(:new)
|
73
|
+
.with(name, id, :candidate, redis, statsd)
|
74
|
+
.and_return(candidate)
|
75
|
+
end
|
76
|
+
|
77
|
+
context "and the candidate is available" do
|
78
|
+
before do
|
79
|
+
allow(candidate).to receive(:available?).and_return(true)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "processes the run output with the candidate" do
|
83
|
+
expect(control).to receive(:process_run_output).with(candidate)
|
84
|
+
subject.perform(name, id, control_output, control_duration, type)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
context "but the candidate is unavailable" do
|
89
|
+
before do
|
90
|
+
allow(candidate).to receive(:available?).and_return(false)
|
91
|
+
allow(described_class).to receive(:perform_in)
|
92
|
+
end
|
93
|
+
|
94
|
+
it "does not process the run output" do
|
95
|
+
expect(control).not_to receive(:process_run_output)
|
96
|
+
subject.perform(name, id, control_output, control_duration, type)
|
97
|
+
end
|
98
|
+
|
99
|
+
it "schedules the job to run again later" do
|
100
|
+
args = [name, id, control_output, control_duration]
|
101
|
+
expect(described_class).to receive(:perform_in).with(5, *args, type.to_sym)
|
102
|
+
subject.perform(*args, type)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
require "byebug"
|
2
|
+
require "timecop"
|
3
|
+
require 'sidekiq/testing'
|
4
|
+
Sidekiq::Logging.logger = nil
|
5
|
+
|
6
|
+
ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
|
7
|
+
|
8
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
9
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
10
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause
|
11
|
+
# this file to always be loaded, without a need to explicitly require it in any
|
12
|
+
# files.
|
13
|
+
#
|
14
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
15
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
16
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
17
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
18
|
+
# a separate helper file that requires the additional dependencies and performs
|
19
|
+
# the additional setup, and require it from the spec files that actually need
|
20
|
+
# it.
|
21
|
+
#
|
22
|
+
# The `.rspec` file also contains a few flags that are not defaults but that
|
23
|
+
# users commonly want.
|
24
|
+
#
|
25
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
26
|
+
RSpec.configure do |config|
|
27
|
+
# rspec-expectations config goes here. You can use an alternate
|
28
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
29
|
+
# assertions if you prefer.
|
30
|
+
config.expect_with :rspec do |expectations|
|
31
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
32
|
+
# and `failure_message` of custom matchers include text for helper methods
|
33
|
+
# defined using `chain`, e.g.:
|
34
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
35
|
+
# # => "be bigger than 2 and smaller than 4"
|
36
|
+
# ...rather than:
|
37
|
+
# # => "be bigger than 2"
|
38
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
39
|
+
end
|
40
|
+
|
41
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
42
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
43
|
+
config.mock_with :rspec do |mocks|
|
44
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
45
|
+
# a real object. This is generally recommended, and will default to
|
46
|
+
# `true` in RSpec 4.
|
47
|
+
mocks.verify_partial_doubles = true
|
48
|
+
end
|
49
|
+
|
50
|
+
# The settings below are suggested to provide a good initial experience
|
51
|
+
# with RSpec, but feel free to customize to your heart's content.
|
52
|
+
|
53
|
+
# These two settings work together to allow you to limit a spec run
|
54
|
+
# to individual examples or groups you care about by tagging them with
|
55
|
+
# `:focus` metadata. When nothing is tagged with `:focus`, all examples
|
56
|
+
# get run.
|
57
|
+
config.filter_run :focus
|
58
|
+
config.run_all_when_everything_filtered = true
|
59
|
+
|
60
|
+
# Allows RSpec to persist some state between runs in order to support
|
61
|
+
# the `--only-failures` and `--next-failure` CLI options. We recommend
|
62
|
+
# you configure your source control system to ignore this file.
|
63
|
+
config.example_status_persistence_file_path = "spec/examples.txt"
|
64
|
+
|
65
|
+
# Limits the available syntax to the non-monkey patched syntax that is
|
66
|
+
# recommended. For more details, see:
|
67
|
+
# - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
|
68
|
+
# - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
69
|
+
# - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
|
70
|
+
config.disable_monkey_patching!
|
71
|
+
|
72
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
73
|
+
# be too noisy due to issues in dependencies.
|
74
|
+
config.warnings = true
|
75
|
+
|
76
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
77
|
+
# file, and it's useful to allow more verbose output when running an
|
78
|
+
# individual spec file.
|
79
|
+
if config.files_to_run.one?
|
80
|
+
# Use the documentation formatter for detailed output,
|
81
|
+
# unless a formatter has already been configured
|
82
|
+
# (e.g. via a command-line flag).
|
83
|
+
config.default_formatter = 'doc'
|
84
|
+
end
|
85
|
+
|
86
|
+
# Run specs in random order to surface order dependencies. If you find an
|
87
|
+
# order dependency and want to debug it, you can fix the order by providing
|
88
|
+
# the seed, which is printed after each run.
|
89
|
+
# --seed 1234
|
90
|
+
config.order = :random
|
91
|
+
|
92
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
93
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
94
|
+
# test failures related to randomization by passing the same `--seed` value
|
95
|
+
# as the one that triggered the failure.
|
96
|
+
Kernel.srand config.seed
|
97
|
+
|
98
|
+
config.before(:each) do
|
99
|
+
Sidekiq::Worker.clear_all
|
100
|
+
end
|
101
|
+
end
|
data/spec/util_spec.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "async_experiments/util"
|
3
|
+
|
4
|
+
RSpec.describe AsyncExperiments::Util do
|
5
|
+
describe ".present?(object)" do
|
6
|
+
it "returns true unless empty or nil" do
|
7
|
+
expect(described_class.present?("This is a test")).to eq(true)
|
8
|
+
expect(described_class.present?([1, 2])).to eq(true)
|
9
|
+
expect(described_class.present?({a: 1})).to eq(true)
|
10
|
+
|
11
|
+
expect(described_class.present?(nil)).to eq(false)
|
12
|
+
expect(described_class.present?("")).to eq(false)
|
13
|
+
expect(described_class.present?([])).to eq(false)
|
14
|
+
expect(described_class.present?({})).to eq(false)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe ".blank?(object)" do
|
19
|
+
it "returns true if empty or nil" do
|
20
|
+
expect(described_class.blank?(nil)).to eq(true)
|
21
|
+
expect(described_class.blank?("")).to eq(true)
|
22
|
+
expect(described_class.blank?([])).to eq(true)
|
23
|
+
expect(described_class.blank?({})).to eq(true)
|
24
|
+
|
25
|
+
expect(described_class.blank?("This is a test")).to eq(false)
|
26
|
+
expect(described_class.blank?([1, 2])).to eq(false)
|
27
|
+
expect(described_class.blank?({a: 1})).to eq(false)
|
28
|
+
expect(described_class.blank?(5)).to eq(false)
|
29
|
+
expect(described_class.blank?(5.5)).to eq(false)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe ".deep_symbolize_keys(hash)" do
|
34
|
+
it "changes top level string keys to symbols" do
|
35
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
36
|
+
"test" => 1,
|
37
|
+
)
|
38
|
+
|
39
|
+
expect(symbolized_hash).to eq(
|
40
|
+
test: 1,
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "changes lower level string keys to symbols" do
|
45
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
46
|
+
test: {
|
47
|
+
"test" => 1,
|
48
|
+
},
|
49
|
+
)
|
50
|
+
|
51
|
+
expect(symbolized_hash).to eq(
|
52
|
+
test: {
|
53
|
+
test: 1,
|
54
|
+
},
|
55
|
+
)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "leaves existing symbol keys alone" do
|
59
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
60
|
+
test: 1,
|
61
|
+
)
|
62
|
+
|
63
|
+
expect(symbolized_hash).to eq(
|
64
|
+
test: 1,
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "leaves other object keys alone" do
|
69
|
+
symbolized_hash = described_class.deep_symbolize_keys(
|
70
|
+
1 => "test",
|
71
|
+
)
|
72
|
+
|
73
|
+
expect(symbolized_hash).to eq(
|
74
|
+
1 => "test",
|
75
|
+
)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
metadata
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: async_experiments
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elliot Crosby-McCullough
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-06-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: hashdiff
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3.4'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3.4'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '11.2'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '11.2'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.10'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.10'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: gem_publisher
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 1.5.0
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.5.0
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: timecop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sidekiq
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description:
|
126
|
+
email:
|
127
|
+
- elliot.cm@gmail.com
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- LICENCE.txt
|
133
|
+
- README.md
|
134
|
+
- lib/async_experiments.rb
|
135
|
+
- lib/async_experiments/candidate_worker.rb
|
136
|
+
- lib/async_experiments/experiment_control.rb
|
137
|
+
- lib/async_experiments/experiment_error_worker.rb
|
138
|
+
- lib/async_experiments/experiment_result.rb
|
139
|
+
- lib/async_experiments/experiment_result_worker.rb
|
140
|
+
- lib/async_experiments/util.rb
|
141
|
+
- lib/async_experiments/version.rb
|
142
|
+
- spec/async_experiments_spec.rb
|
143
|
+
- spec/candidate_worker_spec.rb
|
144
|
+
- spec/examples.txt
|
145
|
+
- spec/experiment_control_spec.rb
|
146
|
+
- spec/experiment_error_worker_spec.rb
|
147
|
+
- spec/experiment_result_spec.rb
|
148
|
+
- spec/experiment_result_worker_spec.rb
|
149
|
+
- spec/spec_helper.rb
|
150
|
+
- spec/util_spec.rb
|
151
|
+
homepage: http://github.com/alphagov/async_experiments
|
152
|
+
licenses:
|
153
|
+
- MIT
|
154
|
+
metadata: {}
|
155
|
+
post_install_message:
|
156
|
+
rdoc_options: []
|
157
|
+
require_paths:
|
158
|
+
- lib
|
159
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">="
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: '0'
|
169
|
+
requirements: []
|
170
|
+
rubyforge_project:
|
171
|
+
rubygems_version: 2.5.1
|
172
|
+
signing_key:
|
173
|
+
specification_version: 4
|
174
|
+
summary: An asynchronous experiment framework.
|
175
|
+
test_files:
|
176
|
+
- spec/experiment_result_spec.rb
|
177
|
+
- spec/spec_helper.rb
|
178
|
+
- spec/experiment_error_worker_spec.rb
|
179
|
+
- spec/experiment_result_worker_spec.rb
|
180
|
+
- spec/async_experiments_spec.rb
|
181
|
+
- spec/util_spec.rb
|
182
|
+
- spec/experiment_control_spec.rb
|
183
|
+
- spec/candidate_worker_spec.rb
|
184
|
+
- spec/examples.txt
|