async_experiments 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 73d448d952cbbee2567c807bafd327fead90ac38
4
+ data.tar.gz: 47f6da2a76c6f75c42ea8904c72421164bc77c3c
5
+ SHA512:
6
+ metadata.gz: c3d83b96b7143a1594c36d3d29b3ba77e6dd65409a35201dae68059cfdc8e2ffe4e03e28b6805d58251a82340984daee93eaf1ec40c03c820caaaf0e85b28ca2
7
+ data.tar.gz: a8932415d200081e7a6176175cbd29372a5e91463cdc040e5d72a5424939af9e10176ef149efa891868af3e4c84cc009ffe06fd5bf99f5ebdf55ee24b47c0100
data/LICENCE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 Crown Copyright (Government Digital Service)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,127 @@
1
+ # Asynchronous Experiments
2
+
3
+ Similar to GitHub Scientist, but uses Sidekiq (and its Redis connection pool)
4
+ to run control and candidate branches of experiments in parallel, storing the
5
+ output for later review.
6
+
7
+ Provides helpers to assist with rendering the output from the comparison.
8
+
9
+ ## IMPORTANT NOTE ABOUT SIDEKIQ
10
+
11
+ This gem expects Sidekiq to be included, but does not list it as a gem dependency.
12
+
13
+ This is because GOV.UK uses `govuk_sidekiq`, a gem which automatically configures
14
+ our apps with some standard Sidekiq configurations. We haven't added that to the
15
+ gemspec as other organisations may want to use other methods to configure Sidekiq.
16
+
17
+ The gem also assumes access to statsd for reporting purposes.
18
+
19
+ ## Technical documentation
20
+
21
+ Example usage:
22
+ ```
23
+ require "async_experiments/experiment_control"
24
+
25
+ class ContentItemsController < ApplicationController
26
+ include AsyncExperiments::ExperimentControl
27
+
28
+ def linkables
29
+ candidate = {
30
+ worker: LinkablesCandidate,
31
+ args: [
32
+ query_params.fetch(:document_type),
33
+ ],
34
+ }
35
+
36
+ presented = experiment_control(:linkables, candidate: candidate) {
37
+ Queries::GetContentCollection.new(
38
+ document_type: query_params.fetch(:document_type),
39
+ fields: %w(
40
+ title
41
+ content_id
42
+ publication_state
43
+ base_path
44
+ internal_name
45
+ ),
46
+ pagination: NullPagination.new
47
+ ).call
48
+ }
49
+
50
+ render json: presented
51
+ end
52
+ end
53
+ ```
54
+
55
+ ```
56
+ require "async_experiments/candidate_worker"
57
+
58
+ class LinkablesCandidate < AsyncExperiments::CandidateWorker
59
+ def perform(document_type, experiment)
60
+ experiment_candidate(experiment) do
61
+ Queries::GetLinkables.new(
62
+ document_type: document_type,
63
+ ).call
64
+ end
65
+ end
66
+ end
67
+ ```
68
+
69
+ ```
70
+ class DebugController < ApplicationController
71
+ skip_before_action :require_signin_permission!
72
+ before_action :validate_experiment_name, only: [:experiment]
73
+
74
+ def experiment
75
+ @mismatched_responses = AsyncExperiments.get_experiment_data(params[:experiment])
76
+ end
77
+
78
+ private
79
+
80
+ def validate_experiment_name
81
+ raise "Experiment names don't contain `:`" if params[:experiment].include?(":")
82
+ end
83
+ end
84
+ ```
85
+
86
+ ```
87
+ <ul>
88
+ <% @mismatched_responses.each_with_index do |mismatch, i| %>
89
+ <li>
90
+ <ul>
91
+ <li><a href="#missing-#{i}">Missing</a></li>
92
+ <li><a href="#extra-#{i}">Extra</a></li>
93
+ <li><a href="#changed-#{i}">Changed</a></li>
94
+ </ul>
95
+
96
+ <h3 id="missing-#{i}">Missing from candidate</h3>
97
+ <% mismatch[:missing].each do |entry| %>
98
+ <pre><%= PP.pp(entry, "") %></pre>
99
+ <% end %>
100
+
101
+ <h3 id="extra-#{i}">Extra in candidate</h3>
102
+ <% mismatch[:extra].each do |entry| %>
103
+ <pre><%= PP.pp(entry, "") %></pre>
104
+ <% end %>
105
+
106
+ <h3 id="changed-#{i}">Changed in candidate</h3>
107
+ <% mismatch[:changed].each do |entry| %>
108
+ <pre><%= PP.pp(entry, "") %></pre>
109
+ <% end %>
110
+ </li>
111
+ <% end %>
112
+ </ul>
113
+ ```
114
+
115
+ ```
116
+ statsd_client = Statsd.new("localhost")
117
+ statsd_client.namespace = "govuk.app.publishing-api"
118
+ AsyncExperiments.statsd = statsd_client
119
+ ```
120
+
121
+ ## Licence
122
+
123
+ [MIT License](LICENCE)
124
+
125
+ ## Versioning policy
126
+
127
+ See https://github.com/alphagov/styleguides/blob/master/rubygems.md#versioning
@@ -0,0 +1,29 @@
1
+ require "async_experiments/experiment_result_worker"
2
+ require "async_experiments/experiment_error_worker"
3
+
4
+ module AsyncExperiments
5
+ class CandidateWorker
6
+ include Sidekiq::Worker
7
+
8
+ sidekiq_options queue: :experiments
9
+
10
+ def experiment_candidate(experiment_config)
11
+ experiment = experiment_config.symbolize_keys
12
+
13
+ start_time = Time.now
14
+ run_output = yield
15
+ duration = (Time.now - start_time).to_f
16
+ ExperimentResultWorker.perform_async(experiment[:name], experiment[:id], run_output, duration, :candidate)
17
+
18
+ run_output
19
+ rescue StandardError => exception
20
+ if ENV["RAISE_EXPERIMENT_ERRORS"]
21
+ raise exception
22
+ else
23
+ backtrace = exception.backtrace
24
+ backtrace.unshift(exception.inspect)
25
+ ExperimentErrorWorker.perform_async(experiment[:name], backtrace.join("\n"))
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,27 @@
1
+ require "async_experiments/experiment_result_worker"
2
+
3
+ module AsyncExperiments
4
+ module ExperimentControl
5
+ def experiment_control(name, candidate:)
6
+ start_time = Time.now
7
+ run_output = yield
8
+ duration = (Time.now - start_time).to_f
9
+
10
+ id = SecureRandom.uuid
11
+
12
+ if run_output.class == Enumerator
13
+ run_output = run_output.to_a
14
+ end
15
+
16
+ ExperimentResultWorker.perform_async(name, id, run_output, duration, :control)
17
+
18
+ candidate_worker = candidate.fetch(:worker)
19
+ candidate_worker.perform_async(*candidate.fetch(:args),
20
+ name: name,
21
+ id: id,
22
+ )
23
+
24
+ run_output
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,14 @@
1
+ module AsyncExperiments
2
+ class ExperimentErrorWorker
3
+ include Sidekiq::Worker
4
+
5
+ sidekiq_options queue: :experiments
6
+
7
+ def perform(experiment_name, exception_string)
8
+ Sidekiq.redis do |redis|
9
+ AsyncExperiments.statsd.increment("experiments.#{experiment_name}.exceptions")
10
+ redis.rpush("experiments:#{experiment_name}:exceptions", exception_string)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,90 @@
1
+ require "json"
2
+ require "hashdiff"
3
+ require "async_experiments/util"
4
+
5
+ module AsyncExperiments
6
+ class ExperimentResult
7
+ def initialize(name, id, type, redis, statsd, run_output = nil, duration = nil)
8
+ @name = name
9
+ @key = "#{name}:#{id}"
10
+ @redis = redis
11
+ @statsd = statsd
12
+ @type = type
13
+ @run_output = run_output
14
+ @duration = duration
15
+
16
+ if Util.blank?(run_output) || Util.blank?(duration)
17
+ redis_data = data_from_redis
18
+
19
+ if redis_data
20
+ @run_output ||= redis_data.fetch(:run_output)
21
+ @duration ||= redis_data.fetch(:duration)
22
+ end
23
+ end
24
+ end
25
+
26
+ attr_reader :key, :run_output, :duration
27
+
28
+ def store_run_output
29
+ redis.set("experiments:#{key}:#{type}", {
30
+ run_output: run_output,
31
+ duration: duration,
32
+ }.to_json)
33
+ end
34
+
35
+ def process_run_output(candidate)
36
+ variation = HashDiff.diff(sort(self.run_output), sort(candidate.run_output))
37
+ report_data(variation, candidate)
38
+ redis.del("experiments:#{key}:candidate")
39
+ end
40
+
41
+ def control?
42
+ type == :control
43
+ end
44
+
45
+ def candidate?
46
+ type == :candidate
47
+ end
48
+
49
+ def available?
50
+ Util.present?(run_output) && Util.present?(duration)
51
+ end
52
+
53
+ protected
54
+
55
+ attr_reader :redis, :statsd, :type, :name
56
+
57
+ private
58
+
59
+ def data_from_redis
60
+ redis_data = redis.get("experiments:#{key}:#{type}")
61
+
62
+ if Util.present?(redis_data)
63
+ Util.deep_symbolize_keys(JSON.parse(redis_data))
64
+ end
65
+ end
66
+
67
+ def report_data(variation, candidate)
68
+ statsd.timing("experiments.#{name}.control", self.duration)
69
+ statsd.timing("experiments.#{name}.candidate", candidate.duration)
70
+
71
+ if variation != []
72
+ statsd.increment("experiments.#{name}.mismatches")
73
+ redis.rpush("experiments:#{name}:mismatches", JSON.dump(variation))
74
+ end
75
+ end
76
+
77
+ def sort(object)
78
+ case object
79
+ when Array
80
+ object.sort_by(&:object_id)
81
+ when Hash
82
+ object.each_with_object({}) { |(key, value), hash|
83
+ hash[key] = sort(value)
84
+ }
85
+ else
86
+ object
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,36 @@
1
+ require "async_experiments/experiment_result"
2
+
3
+ module AsyncExperiments
4
+ class ExperimentResultWorker
5
+ include Sidekiq::Worker
6
+
7
+ sidekiq_options queue: :experiments
8
+
9
+ LOCK_TIMEOUT = 60
10
+
11
+ def perform(name, id, run_output, duration, type)
12
+ type = type.to_sym
13
+
14
+ Sidekiq.redis do |redis|
15
+ this_branch = ExperimentResult.new(name, id, type, redis, statsd, run_output, duration)
16
+
17
+ if this_branch.control?
18
+ candidate = ExperimentResult.new(name, id, :candidate, redis, statsd)
19
+ if candidate.available?
20
+ this_branch.process_run_output(candidate)
21
+ else
22
+ self.class.perform_in(5, name, id, run_output, duration, type)
23
+ end
24
+ elsif this_branch.candidate?
25
+ this_branch.store_run_output
26
+ end
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ def statsd
33
+ AsyncExperiments.statsd
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,20 @@
1
+ module AsyncExperiments
2
+ module Util
3
+ def self.present?(object)
4
+ !self.blank?(object)
5
+ end
6
+
7
+ def self.blank?(object)
8
+ object.nil? || (object.respond_to?(:empty?) && object.empty?)
9
+ end
10
+
11
+ def self.deep_symbolize_keys(hash)
12
+ return hash unless hash.is_a?(Hash)
13
+
14
+ hash.each_with_object({}) do |(key, value), new_hash|
15
+ key = key.respond_to?(:to_sym) ? key.to_sym : key
16
+ new_hash[key] = self.deep_symbolize_keys(value)
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,3 @@
1
+ module AsyncExperiments
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,51 @@
1
+ require "json"
2
+ require "securerandom"
3
+ require "async_experiments/experiment_result_worker"
4
+
5
+ module AsyncExperiments
6
+ def self.statsd
7
+ @statsd
8
+ end
9
+
10
+ def self.statsd=(statsd)
11
+ @statsd = statsd
12
+ end
13
+
14
+ def self.get_experiment_data(experiment_name)
15
+ mismatched_responses = Sidekiq.redis { |redis|
16
+ redis.lrange("experiments:#{experiment_name}:mismatches", 0, -1)
17
+ }
18
+
19
+ mismatched_responses.map { |json|
20
+ parsed = JSON.parse(json)
21
+
22
+ missing, other = parsed.partition {|(operator, _, _)|
23
+ operator == "-"
24
+ }
25
+
26
+ extra, changed = other.partition {|(operator, _, _)|
27
+ operator == "+"
28
+ }
29
+
30
+ missing_entries, extra_entries = self.fix_ordering_issues(
31
+ missing.map(&:last),
32
+ extra.map(&:last),
33
+ )
34
+
35
+ {
36
+ missing: missing_entries,
37
+ extra: extra_entries,
38
+ changed: changed.map(&:last),
39
+ }
40
+ }
41
+ end
42
+
43
+ def self.fix_ordering_issues(missing_entries, extra_entries)
44
+ duplicate_entries = missing_entries & extra_entries
45
+
46
+ missing_entries = missing_entries.reject { |entry| duplicate_entries.include?(entry) }
47
+ extra_entries = extra_entries.reject { |entry| duplicate_entries.include?(entry) }
48
+
49
+ [missing_entries, extra_entries]
50
+ end
51
+ end
@@ -0,0 +1,49 @@
1
+ require "spec_helper"
2
+ require "json"
3
+ require "async_experiments"
4
+
5
+ RSpec.describe AsyncExperiments do
6
+ describe ".get_experiment_data(experiment_name)" do
7
+ let(:name) { "some_experiment" }
8
+
9
+ let(:experiment_results) {
10
+ [
11
+ JSON.dump([
12
+ ["-", "[1]", {same_key: 1, different_key: 2}],
13
+ ["+", "[2]", {same_key: 1, different_key: 3}],
14
+ ["+", "[3]", "Extra element"],
15
+ ["-", "[4]", "Missing element"],
16
+ ["~", "[5]", "Changed element"],
17
+ ["-", "[3]", {moved_complex_object: 1}],
18
+ ["+", "[6]", {moved_complex_object: 1}],
19
+ ])
20
+ ]
21
+ }
22
+
23
+ let(:redis) { double(:redis) }
24
+
25
+ before do
26
+ allow(Sidekiq).to receive(:redis).and_yield(redis)
27
+ allow(redis).to receive(:lrange).with("experiments:#{name}:mismatches", 0, -1)
28
+ .and_return(experiment_results)
29
+ end
30
+
31
+ it "partitions and resorts experiment results for useful output" do
32
+ results = described_class.get_experiment_data(name)
33
+
34
+ expect(results).to eq([
35
+ missing: [
36
+ {"same_key" => 1, "different_key" => 2},
37
+ "Missing element",
38
+ ],
39
+ extra: [
40
+ {"same_key" => 1, "different_key" => 3},
41
+ "Extra element",
42
+ ],
43
+ changed: [
44
+ "Changed element",
45
+ ],
46
+ ])
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,96 @@
1
+ require "spec_helper"
2
+ require "async_experiments/candidate_worker"
3
+ require "async_experiments/experiment_result_worker"
4
+
5
+ RSpec.describe AsyncExperiments::CandidateWorker do
6
+ let(:name) { "some_experiment" }
7
+ let(:id) { SecureRandom.uuid }
8
+
9
+ let(:run_output) { [{some: "output"}] }
10
+
11
+ class TestWorker < AsyncExperiments::CandidateWorker
12
+ def perform(run_output, experiment_config)
13
+ experiment_candidate(experiment_config) do
14
+ Timecop.travel(Time.now + 1.5)
15
+ run_output
16
+ end
17
+ end
18
+ end
19
+
20
+ subject { TestWorker.new }
21
+
22
+ it "uses the 'experiments' queue" do
23
+ Sidekiq::Testing.fake! do
24
+ TestWorker.perform_async
25
+
26
+ expect(TestWorker.jobs.size).to eq(1)
27
+ end
28
+ end
29
+
30
+
31
+ it "returns the control run output" do
32
+ output = subject.perform(run_output,
33
+ name: name,
34
+ id: id,
35
+ )
36
+
37
+ expect(output).to eq(run_output)
38
+ end
39
+
40
+ describe "#experiment_candidate(experiment_config)" do
41
+ it "triggers an ExperimentResultWorker with the candidate output and duration" do
42
+ expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
43
+ .with(name, id, run_output, instance_of(Float), :candidate)
44
+
45
+ subject.perform(run_output,
46
+ name: name,
47
+ id: id,
48
+ )
49
+ end
50
+
51
+ context "when experiment errors are being raised" do
52
+ around do |example|
53
+ setting = ENV["RAISE_EXPERIMENT_ERRORS"]
54
+ ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
55
+
56
+ example.run
57
+
58
+ ENV["RAISE_EXPERIMENT_ERRORS"] = setting
59
+ end
60
+
61
+ it "re-raises experiment errors" do
62
+ allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
63
+
64
+ expect {
65
+ subject.perform(run_output,
66
+ "name" => name,
67
+ "id" => id,
68
+ )
69
+ }.to raise_error(Exception, "Test exception")
70
+ end
71
+ end
72
+
73
+ context "when experiment errors are being quietly reported" do
74
+ around do |example|
75
+ setting = ENV["RAISE_EXPERIMENT_ERRORS"]
76
+ ENV["RAISE_EXPERIMENT_ERRORS"] = nil
77
+
78
+ example.run
79
+
80
+ ENV["RAISE_EXPERIMENT_ERRORS"] = setting
81
+ end
82
+
83
+ it "re-raises experiment errors" do
84
+ allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
85
+
86
+ expect(AsyncExperiments::ExperimentErrorWorker).to receive(:perform_async)
87
+ .with(name, instance_of(String))
88
+
89
+ subject.perform(run_output,
90
+ name: name,
91
+ id: id,
92
+ )
93
+ end
94
+ end
95
+ end
96
+ end
data/spec/examples.txt ADDED
@@ -0,0 +1,37 @@
1
+ example_id | status | run_time |
2
+ ------------------------------------------------ | ------ | --------------- |
3
+ ./spec/async_experiments_spec.rb[1:1:1] | passed | 0.00062 seconds |
4
+ ./spec/candidate_worker_spec.rb[1:1] | passed | 0.00023 seconds |
5
+ ./spec/candidate_worker_spec.rb[1:2] | passed | 0.00053 seconds |
6
+ ./spec/candidate_worker_spec.rb[1:3:1] | passed | 0.00069 seconds |
7
+ ./spec/candidate_worker_spec.rb[1:3:2:1] | passed | 0.00209 seconds |
8
+ ./spec/candidate_worker_spec.rb[1:3:3:1] | passed | 0.0008 seconds |
9
+ ./spec/experiment_control_spec.rb[1:1:1] | passed | 0.00089 seconds |
10
+ ./spec/experiment_control_spec.rb[1:1:2] | passed | 0.00117 seconds |
11
+ ./spec/experiment_control_spec.rb[1:1:3] | passed | 0.00061 seconds |
12
+ ./spec/experiment_error_worker_spec.rb[1:1] | passed | 0.00266 seconds |
13
+ ./spec/experiment_error_worker_spec.rb[1:2] | passed | 0.00056 seconds |
14
+ ./spec/experiment_error_worker_spec.rb[1:3] | passed | 0.0077 seconds |
15
+ ./spec/experiment_result_spec.rb[1:1:1] | passed | 0.00042 seconds |
16
+ ./spec/experiment_result_spec.rb[1:2:1] | passed | 0.00043 seconds |
17
+ ./spec/experiment_result_spec.rb[1:3:1] | passed | 0.00071 seconds |
18
+ ./spec/experiment_result_spec.rb[1:3:2] | passed | 0.00045 seconds |
19
+ ./spec/experiment_result_spec.rb[1:3:3:1] | passed | 0.00055 seconds |
20
+ ./spec/experiment_result_spec.rb[1:3:3:2] | passed | 0.00048 seconds |
21
+ ./spec/experiment_result_spec.rb[1:3:4:1] | passed | 0.00043 seconds |
22
+ ./spec/experiment_result_spec.rb[1:3:4:2] | passed | 0.00058 seconds |
23
+ ./spec/experiment_result_spec.rb[1:4:1:1] | passed | 0.00037 seconds |
24
+ ./spec/experiment_result_spec.rb[1:4:2:1:1] | passed | 0.00095 seconds |
25
+ ./spec/experiment_result_spec.rb[1:4:2:1:2] | passed | 0.00055 seconds |
26
+ ./spec/experiment_result_spec.rb[1:4:2:2:1] | passed | 0.0005 seconds |
27
+ ./spec/experiment_result_worker_spec.rb[1:1] | passed | 0.00048 seconds |
28
+ ./spec/experiment_result_worker_spec.rb[1:2:1] | passed | 0.00175 seconds |
29
+ ./spec/experiment_result_worker_spec.rb[1:3:1:1] | passed | 0.0013 seconds |
30
+ ./spec/experiment_result_worker_spec.rb[1:3:2:1] | passed | 0.00235 seconds |
31
+ ./spec/experiment_result_worker_spec.rb[1:3:2:2] | passed | 0.0025 seconds |
32
+ ./spec/util_spec.rb[1:1:1] | passed | 0.00021 seconds |
33
+ ./spec/util_spec.rb[1:2:1] | passed | 0.00016 seconds |
34
+ ./spec/util_spec.rb[1:3:1] | passed | 0.00014 seconds |
35
+ ./spec/util_spec.rb[1:3:2] | passed | 0.00016 seconds |
36
+ ./spec/util_spec.rb[1:3:3] | passed | 0.00012 seconds |
37
+ ./spec/util_spec.rb[1:3:4] | passed | 0.00011 seconds |
@@ -0,0 +1,60 @@
1
+ require "spec_helper"
2
+ require "async_experiments/experiment_control"
3
+ require "async_experiments/experiment_result_worker"
4
+
5
+ RSpec.describe AsyncExperiments::ExperimentControl do
6
+ let(:name) { :some_experiment }
7
+ let(:id) { double(:id) }
8
+
9
+ let(:candidate_args) { [1, 2, 3] }
10
+ let(:candidate_worker) { double(:candidate_worker, perform_async: nil) }
11
+ let(:candidate_config) {
12
+ {
13
+ worker: candidate_worker,
14
+ args: candidate_args,
15
+ }
16
+ }
17
+
18
+ let(:run_output) { [{some: "output"}] }
19
+
20
+ class TestClass
21
+ include AsyncExperiments::ExperimentControl
22
+
23
+ def call(name, run_output, candidate_config)
24
+ experiment_control(name, candidate: candidate_config) do
25
+ Timecop.travel(Time.now + 1.5)
26
+ run_output
27
+ end
28
+ end
29
+ end
30
+
31
+ subject { TestClass.new }
32
+
33
+ before do
34
+ allow(SecureRandom).to receive(:uuid).and_return(id)
35
+ end
36
+
37
+ describe "#experiment_control(name, candidate: candidate_config)" do
38
+ it "triggers an ExperimentResultWorker with the control output and duration" do
39
+ expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
40
+ .with(name, id, run_output, instance_of(Float), :control)
41
+
42
+ subject.call(name, run_output, candidate_config)
43
+ end
44
+
45
+ it "triggers the candidate worker with its arguments and the experiment config" do
46
+ expect(candidate_worker).to receive(:perform_async)
47
+ .with(*candidate_args,
48
+ name: name,
49
+ id: id,
50
+ )
51
+
52
+ subject.call(name, run_output, candidate_config)
53
+ end
54
+
55
+ it "returns the control run output" do
56
+ output = subject.call(name, run_output, candidate_config)
57
+ expect(output).to eq(run_output)
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,36 @@
1
+ require "spec_helper"
2
+ require "async_experiments"
3
+ require "async_experiments/experiment_error_worker"
4
+
5
+ RSpec.describe AsyncExperiments::ExperimentErrorWorker do
6
+ let(:name) { "some_experiment" }
7
+ let(:error) { "Something went wrong" }
8
+
9
+ let(:statsd) { double(:statsd, increment: nil) }
10
+ let(:redis) { double(:redis, rpush: nil) }
11
+
12
+ subject { described_class.new }
13
+
14
+ before do
15
+ allow(Sidekiq).to receive(:redis).and_yield(redis)
16
+ AsyncExperiments.statsd = statsd
17
+ end
18
+
19
+ it "uses the 'experiments' queue" do
20
+ Sidekiq::Testing.fake! do
21
+ described_class.perform_async
22
+
23
+ expect(described_class.jobs.size).to eq(1)
24
+ end
25
+ end
26
+
27
+ it "increments the statsd error count for the experiment" do
28
+ expect(statsd).to receive(:increment).with("experiments.#{name}.exceptions")
29
+ subject.perform(name, error)
30
+ end
31
+
32
+ it "stores the exception for later reporting" do
33
+ expect(redis).to receive(:rpush).with("experiments:#{name}:exceptions", error)
34
+ subject.perform(name, error)
35
+ end
36
+ end
@@ -0,0 +1,136 @@
1
+ require "spec_helper"
2
+ require "async_experiments/experiment_result"
3
+
4
+ RSpec.describe AsyncExperiments::ExperimentResult do
5
+ let(:id) { SecureRandom.uuid }
6
+ let(:name) { :test_experiment }
7
+
8
+ let(:control_run_output) { "control output" }
9
+ let(:control_duration) { 10.0 }
10
+
11
+ let(:candidate_run_output) { "candidate output" }
12
+ let(:candidate_duration) { 5.0 }
13
+
14
+ let(:redis) { double(:redis, del: nil) }
15
+ let(:statsd) { double(:statsd, timing: nil, increment: nil) }
16
+
17
+ let(:control) { described_class.new(name, id, :control, redis, statsd, control_run_output, control_duration) }
18
+ let(:candidate) { described_class.new(name, id, :candidate, redis, statsd, candidate_run_output, candidate_duration) }
19
+
20
+ describe "#key" do
21
+ it "builds a key from the name and ID" do
22
+ expect(control.key).to eq("test_experiment:#{id}")
23
+ end
24
+ end
25
+
26
+ describe "#store_run_output" do
27
+ it "stores the branch's run output and duration" do
28
+ expect(redis).to receive(:set)
29
+ .with("experiments:#{name}:#{id}:candidate", {
30
+ run_output: candidate_run_output,
31
+ duration: candidate_duration,
32
+ }.to_json)
33
+
34
+ candidate.store_run_output
35
+ end
36
+ end
37
+
38
+ describe "control#process_run_output(candidate)" do
39
+ before do
40
+ allow(redis).to receive(:rpush)
41
+ end
42
+
43
+ it "reports the control and candidate durations to statsd" do
44
+ expect(statsd).to receive(:timing)
45
+ .with("experiments.#{name}.control", control_duration)
46
+
47
+ expect(statsd).to receive(:timing)
48
+ .with("experiments.#{name}.candidate", candidate_duration)
49
+
50
+ control.process_run_output(candidate)
51
+ end
52
+
53
+ it "deletes the candidate data from redis" do
54
+ expect(redis).to receive(:del).with("experiments:#{name}:#{id}:candidate")
55
+
56
+ control.process_run_output(candidate)
57
+ end
58
+
59
+ context "if there's variation between the outputs" do
60
+ it "increments the mismatch count in statsd" do
61
+ expect(statsd).to receive(:increment)
62
+ .with("experiments.#{name}.mismatches")
63
+
64
+ control.process_run_output(candidate)
65
+ end
66
+
67
+ it "logs the mismatch to redis" do
68
+ expect(redis).to receive(:rpush).with(
69
+ "experiments:#{name}:mismatches",
70
+ [["~", "", "control output", "candidate output"]].to_json,
71
+ )
72
+
73
+ control.process_run_output(candidate)
74
+ end
75
+ end
76
+
77
+ context "if there's no variation" do
78
+ let(:candidate_run_output) { control_run_output }
79
+
80
+ it "does not increment the mismatch count" do
81
+ expect(statsd).not_to receive(:increment)
82
+ control.process_run_output(candidate)
83
+ end
84
+
85
+ it "does not log the mismatch to redis" do
86
+ expect(redis).not_to receive(:rpush)
87
+ control.process_run_output(candidate)
88
+ end
89
+ end
90
+ end
91
+
92
+ describe ".new" do
93
+ let(:type) { :candidate }
94
+
95
+ context "if duration and output are provided" do
96
+ it "uses those" do
97
+ candidate = described_class.new(name, id, type, redis, statsd, "arbitrary output", 1.23)
98
+ expect(candidate.run_output).to eq("arbitrary output")
99
+ expect(candidate.duration).to eq(1.23)
100
+ end
101
+ end
102
+
103
+ context "if duration and output are not provided" do
104
+ context "and redis has the data" do
105
+ before do
106
+ allow(redis).to receive(:get).with("experiments:#{name}:#{id}:#{type}").and_return({
107
+ run_output: candidate_run_output,
108
+ duration: candidate_duration,
109
+ }.to_json)
110
+ end
111
+
112
+ it "uses the redis data" do
113
+ candidate = described_class.new(name, id, type, redis, statsd)
114
+ expect(candidate.run_output).to eq(candidate_run_output)
115
+ expect(candidate.duration).to eq(candidate_duration)
116
+ end
117
+
118
+ it "is considered available" do
119
+ candidate = described_class.new(name, id, type, redis, statsd)
120
+ expect(candidate.available?).to eq(true)
121
+ end
122
+ end
123
+
124
+ context "but redis does not have the data" do
125
+ before do
126
+ allow(redis).to receive(:get).and_return("")
127
+ end
128
+
129
+ it "is considered unavailable" do
130
+ missing_candidate = described_class.new(name, id, type, redis, statsd)
131
+ expect(missing_candidate.available?).to eq(false)
132
+ end
133
+ end
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,106 @@
1
+ require "spec_helper"
2
+ require "async_experiments"
3
+ require "async_experiments/experiment_result"
4
+ require "async_experiments/experiment_result_worker"
5
+
6
+ RSpec.describe AsyncExperiments::ExperimentResultWorker do
7
+ let(:name) { "some_experiment" }
8
+ let(:id) { SecureRandom.uuid }
9
+
10
+ let(:statsd) { double(:statsd) }
11
+ let(:redis) { double(:redis) }
12
+ let(:redis_key) { double(:redis_key) }
13
+
14
+ let(:control_output) { "control output" }
15
+ let(:control_duration) { 10.0 }
16
+
17
+ let(:candidate_output) { "candidate output" }
18
+ let(:candidate_duration) { 5.0 }
19
+
20
+ let(:control) {
21
+ double(:control,
22
+ control?: true,
23
+ candidate?: false,
24
+ key: redis_key,
25
+ )
26
+ }
27
+ let(:candidate) {
28
+ double(:candidate,
29
+ candidate?: true,
30
+ control?: false,
31
+ key: redis_key,
32
+ )
33
+ }
34
+
35
+ subject { described_class.new }
36
+
37
+ before do
38
+ allow(Sidekiq).to receive(:redis).and_yield(redis)
39
+ AsyncExperiments.statsd = statsd
40
+ end
41
+
42
+ it "uses the 'experiments' queue" do
43
+ Sidekiq::Testing.fake! do
44
+ described_class.perform_async
45
+
46
+ expect(described_class.jobs.size).to eq(1)
47
+ end
48
+ end
49
+
50
+ context "if we're the candidate" do
51
+ let(:type) { "candidate" }
52
+
53
+ it "stores the run output and duration" do
54
+ expect(AsyncExperiments::ExperimentResult).to receive(:new)
55
+ .with(name, id, type.to_sym, redis, statsd, candidate_output, candidate_duration)
56
+ .and_return(candidate)
57
+
58
+ expect(candidate).to receive(:store_run_output)
59
+
60
+ subject.perform(name, id, candidate_output, candidate_duration, type)
61
+ end
62
+ end
63
+
64
+ context "if we're the control" do
65
+ let(:type) { "control" }
66
+
67
+ before do
68
+ expect(AsyncExperiments::ExperimentResult).to receive(:new)
69
+ .with(name, id, :control, redis, statsd, control_output, control_duration)
70
+ .and_return(control)
71
+
72
+ expect(AsyncExperiments::ExperimentResult).to receive(:new)
73
+ .with(name, id, :candidate, redis, statsd)
74
+ .and_return(candidate)
75
+ end
76
+
77
+ context "and the candidate is available" do
78
+ before do
79
+ allow(candidate).to receive(:available?).and_return(true)
80
+ end
81
+
82
+ it "processes the run output with the candidate" do
83
+ expect(control).to receive(:process_run_output).with(candidate)
84
+ subject.perform(name, id, control_output, control_duration, type)
85
+ end
86
+ end
87
+
88
+ context "but the candidate is unavailable" do
89
+ before do
90
+ allow(candidate).to receive(:available?).and_return(false)
91
+ allow(described_class).to receive(:perform_in)
92
+ end
93
+
94
+ it "does not process the run output" do
95
+ expect(control).not_to receive(:process_run_output)
96
+ subject.perform(name, id, control_output, control_duration, type)
97
+ end
98
+
99
+ it "schedules the job to run again later" do
100
+ args = [name, id, control_output, control_duration]
101
+ expect(described_class).to receive(:perform_in).with(5, *args, type.to_sym)
102
+ subject.perform(*args, type)
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,101 @@
1
+ require "byebug"
2
+ require "timecop"
3
+ require 'sidekiq/testing'
4
+ Sidekiq::Logging.logger = nil
5
+
6
+ ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
7
+
8
+ # This file was generated by the `rspec --init` command. Conventionally, all
9
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
10
+ # The generated `.rspec` file contains `--require spec_helper` which will cause
11
+ # this file to always be loaded, without a need to explicitly require it in any
12
+ # files.
13
+ #
14
+ # Given that it is always loaded, you are encouraged to keep this file as
15
+ # light-weight as possible. Requiring heavyweight dependencies from this file
16
+ # will add to the boot time of your test suite on EVERY test run, even for an
17
+ # individual file that may not need all of that loaded. Instead, consider making
18
+ # a separate helper file that requires the additional dependencies and performs
19
+ # the additional setup, and require it from the spec files that actually need
20
+ # it.
21
+ #
22
+ # The `.rspec` file also contains a few flags that are not defaults but that
23
+ # users commonly want.
24
+ #
25
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
26
+ RSpec.configure do |config|
27
+ # rspec-expectations config goes here. You can use an alternate
28
+ # assertion/expectation library such as wrong or the stdlib/minitest
29
+ # assertions if you prefer.
30
+ config.expect_with :rspec do |expectations|
31
+ # This option will default to `true` in RSpec 4. It makes the `description`
32
+ # and `failure_message` of custom matchers include text for helper methods
33
+ # defined using `chain`, e.g.:
34
+ # be_bigger_than(2).and_smaller_than(4).description
35
+ # # => "be bigger than 2 and smaller than 4"
36
+ # ...rather than:
37
+ # # => "be bigger than 2"
38
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
39
+ end
40
+
41
+ # rspec-mocks config goes here. You can use an alternate test double
42
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
43
+ config.mock_with :rspec do |mocks|
44
+ # Prevents you from mocking or stubbing a method that does not exist on
45
+ # a real object. This is generally recommended, and will default to
46
+ # `true` in RSpec 4.
47
+ mocks.verify_partial_doubles = true
48
+ end
49
+
50
+ # The settings below are suggested to provide a good initial experience
51
+ # with RSpec, but feel free to customize to your heart's content.
52
+
53
+ # These two settings work together to allow you to limit a spec run
54
+ # to individual examples or groups you care about by tagging them with
55
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples
56
+ # get run.
57
+ config.filter_run :focus
58
+ config.run_all_when_everything_filtered = true
59
+
60
+ # Allows RSpec to persist some state between runs in order to support
61
+ # the `--only-failures` and `--next-failure` CLI options. We recommend
62
+ # you configure your source control system to ignore this file.
63
+ config.example_status_persistence_file_path = "spec/examples.txt"
64
+
65
+ # Limits the available syntax to the non-monkey patched syntax that is
66
+ # recommended. For more details, see:
67
+ # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
68
+ # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
69
+ # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
70
+ config.disable_monkey_patching!
71
+
72
+ # This setting enables warnings. It's recommended, but in some cases may
73
+ # be too noisy due to issues in dependencies.
74
+ config.warnings = true
75
+
76
+ # Many RSpec users commonly either run the entire suite or an individual
77
+ # file, and it's useful to allow more verbose output when running an
78
+ # individual spec file.
79
+ if config.files_to_run.one?
80
+ # Use the documentation formatter for detailed output,
81
+ # unless a formatter has already been configured
82
+ # (e.g. via a command-line flag).
83
+ config.default_formatter = 'doc'
84
+ end
85
+
86
+ # Run specs in random order to surface order dependencies. If you find an
87
+ # order dependency and want to debug it, you can fix the order by providing
88
+ # the seed, which is printed after each run.
89
+ # --seed 1234
90
+ config.order = :random
91
+
92
+ # Seed global randomization in this process using the `--seed` CLI option.
93
+ # Setting this allows you to use `--seed` to deterministically reproduce
94
+ # test failures related to randomization by passing the same `--seed` value
95
+ # as the one that triggered the failure.
96
+ Kernel.srand config.seed
97
+
98
+ config.before(:each) do
99
+ Sidekiq::Worker.clear_all
100
+ end
101
+ end
data/spec/util_spec.rb ADDED
@@ -0,0 +1,78 @@
1
+ require "spec_helper"
2
+ require "async_experiments/util"
3
+
4
+ RSpec.describe AsyncExperiments::Util do
5
+ describe ".present?(object)" do
6
+ it "returns true unless empty or nil" do
7
+ expect(described_class.present?("This is a test")).to eq(true)
8
+ expect(described_class.present?([1, 2])).to eq(true)
9
+ expect(described_class.present?({a: 1})).to eq(true)
10
+
11
+ expect(described_class.present?(nil)).to eq(false)
12
+ expect(described_class.present?("")).to eq(false)
13
+ expect(described_class.present?([])).to eq(false)
14
+ expect(described_class.present?({})).to eq(false)
15
+ end
16
+ end
17
+
18
+ describe ".blank?(object)" do
19
+ it "returns true if empty or nil" do
20
+ expect(described_class.blank?(nil)).to eq(true)
21
+ expect(described_class.blank?("")).to eq(true)
22
+ expect(described_class.blank?([])).to eq(true)
23
+ expect(described_class.blank?({})).to eq(true)
24
+
25
+ expect(described_class.blank?("This is a test")).to eq(false)
26
+ expect(described_class.blank?([1, 2])).to eq(false)
27
+ expect(described_class.blank?({a: 1})).to eq(false)
28
+ expect(described_class.blank?(5)).to eq(false)
29
+ expect(described_class.blank?(5.5)).to eq(false)
30
+ end
31
+ end
32
+
33
+ describe ".deep_symbolize_keys(hash)" do
34
+ it "changes top level string keys to symbols" do
35
+ symbolized_hash = described_class.deep_symbolize_keys(
36
+ "test" => 1,
37
+ )
38
+
39
+ expect(symbolized_hash).to eq(
40
+ test: 1,
41
+ )
42
+ end
43
+
44
+ it "changes lower level string keys to symbols" do
45
+ symbolized_hash = described_class.deep_symbolize_keys(
46
+ test: {
47
+ "test" => 1,
48
+ },
49
+ )
50
+
51
+ expect(symbolized_hash).to eq(
52
+ test: {
53
+ test: 1,
54
+ },
55
+ )
56
+ end
57
+
58
+ it "leaves existing symbol keys alone" do
59
+ symbolized_hash = described_class.deep_symbolize_keys(
60
+ test: 1,
61
+ )
62
+
63
+ expect(symbolized_hash).to eq(
64
+ test: 1,
65
+ )
66
+ end
67
+
68
+ it "leaves other object keys alone" do
69
+ symbolized_hash = described_class.deep_symbolize_keys(
70
+ 1 => "test",
71
+ )
72
+
73
+ expect(symbolized_hash).to eq(
74
+ 1 => "test",
75
+ )
76
+ end
77
+ end
78
+ end
metadata ADDED
@@ -0,0 +1,184 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: async_experiments
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Elliot Crosby-McCullough
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-06-17 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: hashdiff
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '3.4'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '3.4'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '11.2'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '11.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '1.10'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '1.10'
69
+ - !ruby/object:Gem::Dependency
70
+ name: gem_publisher
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '='
74
+ - !ruby/object:Gem::Version
75
+ version: 1.5.0
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '='
81
+ - !ruby/object:Gem::Version
82
+ version: 1.5.0
83
+ - !ruby/object:Gem::Dependency
84
+ name: byebug
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: timecop
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: sidekiq
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ description:
126
+ email:
127
+ - elliot.cm@gmail.com
128
+ executables: []
129
+ extensions: []
130
+ extra_rdoc_files: []
131
+ files:
132
+ - LICENCE.txt
133
+ - README.md
134
+ - lib/async_experiments.rb
135
+ - lib/async_experiments/candidate_worker.rb
136
+ - lib/async_experiments/experiment_control.rb
137
+ - lib/async_experiments/experiment_error_worker.rb
138
+ - lib/async_experiments/experiment_result.rb
139
+ - lib/async_experiments/experiment_result_worker.rb
140
+ - lib/async_experiments/util.rb
141
+ - lib/async_experiments/version.rb
142
+ - spec/async_experiments_spec.rb
143
+ - spec/candidate_worker_spec.rb
144
+ - spec/examples.txt
145
+ - spec/experiment_control_spec.rb
146
+ - spec/experiment_error_worker_spec.rb
147
+ - spec/experiment_result_spec.rb
148
+ - spec/experiment_result_worker_spec.rb
149
+ - spec/spec_helper.rb
150
+ - spec/util_spec.rb
151
+ homepage: http://github.com/alphagov/async_experiments
152
+ licenses:
153
+ - MIT
154
+ metadata: {}
155
+ post_install_message:
156
+ rdoc_options: []
157
+ require_paths:
158
+ - lib
159
+ required_ruby_version: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - ">="
162
+ - !ruby/object:Gem::Version
163
+ version: '0'
164
+ required_rubygems_version: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - ">="
167
+ - !ruby/object:Gem::Version
168
+ version: '0'
169
+ requirements: []
170
+ rubyforge_project:
171
+ rubygems_version: 2.5.1
172
+ signing_key:
173
+ specification_version: 4
174
+ summary: An asynchronous experiment framework.
175
+ test_files:
176
+ - spec/experiment_result_spec.rb
177
+ - spec/spec_helper.rb
178
+ - spec/experiment_error_worker_spec.rb
179
+ - spec/experiment_result_worker_spec.rb
180
+ - spec/async_experiments_spec.rb
181
+ - spec/util_spec.rb
182
+ - spec/experiment_control_spec.rb
183
+ - spec/candidate_worker_spec.rb
184
+ - spec/examples.txt