async_experiments 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 73d448d952cbbee2567c807bafd327fead90ac38
+  data.tar.gz: 47f6da2a76c6f75c42ea8904c72421164bc77c3c
+SHA512:
+  metadata.gz: c3d83b96b7143a1594c36d3d29b3ba77e6dd65409a35201dae68059cfdc8e2ffe4e03e28b6805d58251a82340984daee93eaf1ec40c03c820caaaf0e85b28ca2
+  data.tar.gz: a8932415d200081e7a6176175cbd29372a5e91463cdc040e5d72a5424939af9e10176ef149efa891868af3e4c84cc009ffe06fd5bf99f5ebdf55ee24b47c0100

data/LICENCE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Crown Copyright (Government Digital Service)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+# Asynchronous Experiments
+
+Similar to GitHub Scientist, but uses Sidekiq (and its Redis connection pool)
+to run control and candidate branches of experiments in parallel, storing the
+output for later review.
+
+Provides helpers to assist with rendering the output from the comparison.
+
+## IMPORTANT NOTE ABOUT SIDEKIQ
+
+This gem expects Sidekiq to be included, but does not list it as a gem dependency.
+
+This is because GOV.UK uses `govuk_sidekiq`, a gem which automatically configures
+our apps with some standard Sidekiq configurations.  We haven't added that to the
+gemspec as other organisations may want to use other methods to configure Sidekiq.
+
+The gem also assumes access to statsd for reporting purposes.
+
+## Technical documentation
+
+Example usage:
+```
+require "async_experiments/experiment_control"
+
+class ContentItemsController < ApplicationController
+  include AsyncExperiments::ExperimentControl
+
+  def linkables
+    candidate = {
+      worker: LinkablesCandidate,
+      args: [
+        query_params.fetch(:document_type),
+      ],
+    }
+
+    presented = experiment_control(:linkables, candidate: candidate) {
+      Queries::GetContentCollection.new(
+        document_type: query_params.fetch(:document_type),
+        fields: %w(
+          title
+          content_id
+          publication_state
+          base_path
+          internal_name
+        ),
+        pagination: NullPagination.new
+      ).call
+    }
+
+    render json: presented
+  end
+end
+```
+
+```
+require "async_experiments/candidate_worker"
+
+class LinkablesCandidate < AsyncExperiments::CandidateWorker
+  def perform(document_type, experiment)
+    experiment_candidate(experiment) do
+      Queries::GetLinkables.new(
+        document_type: document_type,
+      ).call
+    end
+  end
+end
+```
+
+```
+class DebugController < ApplicationController
+  skip_before_action :require_signin_permission!
+  before_action :validate_experiment_name, only: [:experiment]
+
+  def experiment
+    @mismatched_responses = AsyncExperiments.get_experiment_data(params[:experiment])
+  end
+
+private
+
+  def validate_experiment_name
+    raise "Experiment names don't contain `:`" if params[:experiment].include?(":")
+  end
+end
+```
+
+```
+<ul>
+  <% @mismatched_responses.each_with_index do |mismatch, i| %>
+    <li>
+      <ul>
+        <li><a href="#missing-#{i}">Missing</a></li>
+        <li><a href="#extra-#{i}">Extra</a></li>
+        <li><a href="#changed-#{i}">Changed</a></li>
+      </ul>
+
+      <h3 id="missing-#{i}">Missing from candidate</h3>
+      <% mismatch[:missing].each do |entry| %>
+        <pre><%= PP.pp(entry, "") %></pre>
+      <% end %>
+
+      <h3 id="extra-#{i}">Extra in candidate</h3>
+      <% mismatch[:extra].each do |entry| %>
+        <pre><%= PP.pp(entry, "") %></pre>
+      <% end %>
+
+      <h3 id="changed-#{i}">Changed in candidate</h3>
+      <% mismatch[:changed].each do |entry| %>
+        <pre><%= PP.pp(entry, "") %></pre>
+      <% end %>
+    </li>
+  <% end %>
+</ul>
+```
+
+```
+statsd_client = Statsd.new("localhost")
+statsd_client.namespace = "govuk.app.publishing-api"
+AsyncExperiments.statsd = statsd_client
+```
+
+## Licence
+
+[MIT License](LICENCE)
+
+## Versioning policy
+
+See https://github.com/alphagov/styleguides/blob/master/rubygems.md#versioning

data/lib/async_experiments/candidate_worker.rb

@@ -0,0 +1,29 @@
+require "async_experiments/experiment_result_worker"
+require "async_experiments/experiment_error_worker"
+
+module AsyncExperiments
+  class CandidateWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :experiments
+
+    def experiment_candidate(experiment_config)
+      experiment = experiment_config.symbolize_keys
+
+      start_time = Time.now
+      run_output = yield
+      duration = (Time.now - start_time).to_f
+      ExperimentResultWorker.perform_async(experiment[:name], experiment[:id], run_output, duration, :candidate)
+
+      run_output
+    rescue StandardError => exception
+      if ENV["RAISE_EXPERIMENT_ERRORS"]
+        raise exception
+      else
+        backtrace = exception.backtrace
+        backtrace.unshift(exception.inspect)
+        ExperimentErrorWorker.perform_async(experiment[:name], backtrace.join("\n"))
+      end
+    end
+  end
+end

data/lib/async_experiments/experiment_control.rb

@@ -0,0 +1,27 @@
+require "async_experiments/experiment_result_worker"
+
+module AsyncExperiments
+  module ExperimentControl
+    def experiment_control(name, candidate:)
+      start_time = Time.now
+      run_output = yield
+      duration = (Time.now - start_time).to_f
+
+      id = SecureRandom.uuid
+
+      if run_output.class == Enumerator
+        run_output = run_output.to_a
+      end
+
+      ExperimentResultWorker.perform_async(name, id, run_output, duration, :control)
+
+      candidate_worker = candidate.fetch(:worker)
+      candidate_worker.perform_async(*candidate.fetch(:args),
+        name: name,
+        id: id,
+      )
+
+      run_output
+    end
+  end
+end

data/lib/async_experiments/experiment_error_worker.rb

@@ -0,0 +1,14 @@
+module AsyncExperiments
+  class ExperimentErrorWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :experiments
+
+    def perform(experiment_name, exception_string)
+      Sidekiq.redis do |redis|
+        AsyncExperiments.statsd.increment("experiments.#{experiment_name}.exceptions")
+        redis.rpush("experiments:#{experiment_name}:exceptions", exception_string)
+      end
+    end
+  end
+end

data/lib/async_experiments/experiment_result.rb

@@ -0,0 +1,90 @@
+require "json"
+require "hashdiff"
+require "async_experiments/util"
+
+module AsyncExperiments
+  class ExperimentResult
+    def initialize(name, id, type, redis, statsd, run_output = nil, duration = nil)
+      @name = name
+      @key = "#{name}:#{id}"
+      @redis = redis
+      @statsd = statsd
+      @type = type
+      @run_output = run_output
+      @duration = duration
+
+      if Util.blank?(run_output) || Util.blank?(duration)
+        redis_data = data_from_redis
+
+        if redis_data
+          @run_output ||= redis_data.fetch(:run_output)
+          @duration ||= redis_data.fetch(:duration)
+        end
+      end
+    end
+
+    attr_reader :key, :run_output, :duration
+
+    def store_run_output
+      redis.set("experiments:#{key}:#{type}", {
+        run_output: run_output,
+        duration: duration,
+      }.to_json)
+    end
+
+    def process_run_output(candidate)
+      variation = HashDiff.diff(sort(self.run_output), sort(candidate.run_output))
+      report_data(variation, candidate)
+      redis.del("experiments:#{key}:candidate")
+    end
+
+    def control?
+      type == :control
+    end
+
+    def candidate?
+      type == :candidate
+    end
+
+    def available?
+      Util.present?(run_output) && Util.present?(duration)
+    end
+
+  protected
+
+    attr_reader :redis, :statsd, :type, :name
+
+  private
+
+    def data_from_redis
+      redis_data = redis.get("experiments:#{key}:#{type}")
+
+      if Util.present?(redis_data)
+        Util.deep_symbolize_keys(JSON.parse(redis_data))
+      end
+    end
+
+    def report_data(variation, candidate)
+      statsd.timing("experiments.#{name}.control", self.duration)
+      statsd.timing("experiments.#{name}.candidate", candidate.duration)
+
+      if variation != []
+        statsd.increment("experiments.#{name}.mismatches")
+        redis.rpush("experiments:#{name}:mismatches", JSON.dump(variation))
+      end
+    end
+
+    def sort(object)
+      case object
+      when Array
+        object.sort_by(&:object_id)
+      when Hash
+        object.each_with_object({}) { |(key, value), hash|
+          hash[key] = sort(value)
+        }
+      else
+        object
+      end
+    end
+  end
+end

data/lib/async_experiments/experiment_result_worker.rb

@@ -0,0 +1,36 @@
+require "async_experiments/experiment_result"
+
+module AsyncExperiments
+  class ExperimentResultWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :experiments
+
+    LOCK_TIMEOUT = 60
+
+    def perform(name, id, run_output, duration, type)
+      type = type.to_sym
+
+      Sidekiq.redis do |redis|
+        this_branch = ExperimentResult.new(name, id, type, redis, statsd, run_output, duration)
+
+        if this_branch.control?
+          candidate = ExperimentResult.new(name, id, :candidate, redis, statsd)
+          if candidate.available?
+            this_branch.process_run_output(candidate)
+          else
+            self.class.perform_in(5, name, id, run_output, duration, type)
+          end
+        elsif this_branch.candidate?
+          this_branch.store_run_output
+        end
+      end
+    end
+
+  private
+
+    def statsd
+      AsyncExperiments.statsd
+    end
+  end
+end

data/lib/async_experiments/util.rb

@@ -0,0 +1,20 @@
+module AsyncExperiments
+  module Util
+    def self.present?(object)
+      !self.blank?(object)
+    end
+
+    def self.blank?(object)
+      object.nil? || (object.respond_to?(:empty?) && object.empty?)
+    end
+
+    def self.deep_symbolize_keys(hash)
+      return hash unless hash.is_a?(Hash)
+
+      hash.each_with_object({}) do |(key, value), new_hash|
+        key = key.respond_to?(:to_sym) ? key.to_sym : key
+        new_hash[key] = self.deep_symbolize_keys(value)
+      end
+    end
+  end
+end

data/lib/async_experiments/version.rb

@@ -0,0 +1,3 @@
+module AsyncExperiments
+  VERSION = "0.0.1"
+end

data/lib/async_experiments.rb

@@ -0,0 +1,51 @@
+require "json"
+require "securerandom"
+require "async_experiments/experiment_result_worker"
+
+module AsyncExperiments
+  def self.statsd
+    @statsd
+  end
+
+  def self.statsd=(statsd)
+    @statsd = statsd
+  end
+
+  def self.get_experiment_data(experiment_name)
+    mismatched_responses = Sidekiq.redis { |redis|
+      redis.lrange("experiments:#{experiment_name}:mismatches", 0, -1)
+    }
+
+    mismatched_responses.map { |json|
+      parsed = JSON.parse(json)
+
+      missing, other = parsed.partition {|(operator, _, _)|
+        operator == "-"
+      }
+
+      extra, changed = other.partition {|(operator, _, _)|
+        operator == "+"
+      }
+
+      missing_entries, extra_entries = self.fix_ordering_issues(
+        missing.map(&:last),
+        extra.map(&:last),
+      )
+
+      {
+        missing: missing_entries,
+        extra: extra_entries,
+        changed: changed.map(&:last),
+      }
+    }
+  end
+
+  def self.fix_ordering_issues(missing_entries, extra_entries)
+    duplicate_entries = missing_entries & extra_entries
+
+    missing_entries = missing_entries.reject { |entry| duplicate_entries.include?(entry) }
+    extra_entries = extra_entries.reject { |entry| duplicate_entries.include?(entry) }
+
+    [missing_entries, extra_entries]
+  end
+end

data/spec/async_experiments_spec.rb

@@ -0,0 +1,49 @@
+require "spec_helper"
+require "json"
+require "async_experiments"
+
+RSpec.describe AsyncExperiments do
+  describe ".get_experiment_data(experiment_name)" do
+    let(:name) { "some_experiment" }
+
+    let(:experiment_results) {
+      [
+        JSON.dump([
+          ["-", "[1]", {same_key: 1, different_key: 2}],
+          ["+", "[2]", {same_key: 1, different_key: 3}],
+          ["+", "[3]", "Extra element"],
+          ["-", "[4]", "Missing element"],
+          ["~", "[5]", "Changed element"],
+          ["-", "[3]", {moved_complex_object: 1}],
+          ["+", "[6]", {moved_complex_object: 1}],
+        ])
+      ]
+    }
+
+    let(:redis) { double(:redis) }
+
+    before do
+      allow(Sidekiq).to receive(:redis).and_yield(redis)
+      allow(redis).to receive(:lrange).with("experiments:#{name}:mismatches", 0, -1)
+        .and_return(experiment_results)
+    end
+
+    it "partitions and resorts experiment results for useful output" do
+      results = described_class.get_experiment_data(name)
+
+      expect(results).to eq([
+        missing: [
+          {"same_key" => 1, "different_key" => 2},
+          "Missing element",
+        ],
+        extra: [
+          {"same_key" => 1, "different_key" => 3},
+          "Extra element",
+        ],
+        changed: [
+          "Changed element",
+        ],
+      ])
+    end
+  end
+end

data/spec/candidate_worker_spec.rb

@@ -0,0 +1,96 @@
+require "spec_helper"
+require "async_experiments/candidate_worker"
+require "async_experiments/experiment_result_worker"
+
+RSpec.describe AsyncExperiments::CandidateWorker do
+  let(:name) { "some_experiment" }
+  let(:id) { SecureRandom.uuid }
+
+  let(:run_output) { [{some: "output"}] }
+
+  class TestWorker < AsyncExperiments::CandidateWorker
+    def perform(run_output, experiment_config)
+      experiment_candidate(experiment_config) do
+        Timecop.travel(Time.now + 1.5)
+        run_output
+      end
+    end
+  end
+
+  subject { TestWorker.new }
+
+  it "uses the 'experiments' queue" do
+    Sidekiq::Testing.fake! do
+      TestWorker.perform_async
+
+      expect(TestWorker.jobs.size).to eq(1)
+    end
+  end
+
+
+  it "returns the control run output" do
+    output = subject.perform(run_output,
+      name: name,
+      id: id,
+    )
+
+    expect(output).to eq(run_output)
+  end
+
+  describe "#experiment_candidate(experiment_config)" do
+    it "triggers an ExperimentResultWorker with the candidate output and duration" do
+      expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
+        .with(name, id, run_output, instance_of(Float), :candidate)
+
+      subject.perform(run_output,
+        name: name,
+        id: id,
+      )
+    end
+
+    context "when experiment errors are being raised" do
+      around do |example|
+        setting = ENV["RAISE_EXPERIMENT_ERRORS"]
+        ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
+
+        example.run
+
+        ENV["RAISE_EXPERIMENT_ERRORS"] = setting
+      end
+
+      it "re-raises experiment errors" do
+        allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
+
+        expect {
+          subject.perform(run_output,
+            "name" => name,
+            "id" => id,
+          )
+        }.to raise_error(Exception, "Test exception")
+      end
+    end
+
+    context "when experiment errors are being quietly reported" do
+      around do |example|
+        setting = ENV["RAISE_EXPERIMENT_ERRORS"]
+        ENV["RAISE_EXPERIMENT_ERRORS"] = nil
+
+        example.run
+
+        ENV["RAISE_EXPERIMENT_ERRORS"] = setting
+      end
+
+      it "re-raises experiment errors" do
+        allow(Time).to receive(:now).and_raise(StandardError.new("Test exception"))
+
+        expect(AsyncExperiments::ExperimentErrorWorker).to receive(:perform_async)
+          .with(name, instance_of(String))
+
+        subject.perform(run_output,
+          name: name,
+          id: id,
+        )
+      end
+    end
+  end
+end

data/spec/examples.txt

@@ -0,0 +1,37 @@
+example_id                                       | status | run_time        |
+------------------------------------------------ | ------ | --------------- |
+./spec/async_experiments_spec.rb[1:1:1]          | passed | 0.00062 seconds |
+./spec/candidate_worker_spec.rb[1:1]             | passed | 0.00023 seconds |
+./spec/candidate_worker_spec.rb[1:2]             | passed | 0.00053 seconds |
+./spec/candidate_worker_spec.rb[1:3:1]           | passed | 0.00069 seconds |
+./spec/candidate_worker_spec.rb[1:3:2:1]         | passed | 0.00209 seconds |
+./spec/candidate_worker_spec.rb[1:3:3:1]         | passed | 0.0008 seconds  |
+./spec/experiment_control_spec.rb[1:1:1]         | passed | 0.00089 seconds |
+./spec/experiment_control_spec.rb[1:1:2]         | passed | 0.00117 seconds |
+./spec/experiment_control_spec.rb[1:1:3]         | passed | 0.00061 seconds |
+./spec/experiment_error_worker_spec.rb[1:1]      | passed | 0.00266 seconds |
+./spec/experiment_error_worker_spec.rb[1:2]      | passed | 0.00056 seconds |
+./spec/experiment_error_worker_spec.rb[1:3]      | passed | 0.0077 seconds  |
+./spec/experiment_result_spec.rb[1:1:1]          | passed | 0.00042 seconds |
+./spec/experiment_result_spec.rb[1:2:1]          | passed | 0.00043 seconds |
+./spec/experiment_result_spec.rb[1:3:1]          | passed | 0.00071 seconds |
+./spec/experiment_result_spec.rb[1:3:2]          | passed | 0.00045 seconds |
+./spec/experiment_result_spec.rb[1:3:3:1]        | passed | 0.00055 seconds |
+./spec/experiment_result_spec.rb[1:3:3:2]        | passed | 0.00048 seconds |
+./spec/experiment_result_spec.rb[1:3:4:1]        | passed | 0.00043 seconds |
+./spec/experiment_result_spec.rb[1:3:4:2]        | passed | 0.00058 seconds |
+./spec/experiment_result_spec.rb[1:4:1:1]        | passed | 0.00037 seconds |
+./spec/experiment_result_spec.rb[1:4:2:1:1]      | passed | 0.00095 seconds |
+./spec/experiment_result_spec.rb[1:4:2:1:2]      | passed | 0.00055 seconds |
+./spec/experiment_result_spec.rb[1:4:2:2:1]      | passed | 0.0005 seconds  |
+./spec/experiment_result_worker_spec.rb[1:1]     | passed | 0.00048 seconds |
+./spec/experiment_result_worker_spec.rb[1:2:1]   | passed | 0.00175 seconds |
+./spec/experiment_result_worker_spec.rb[1:3:1:1] | passed | 0.0013 seconds  |
+./spec/experiment_result_worker_spec.rb[1:3:2:1] | passed | 0.00235 seconds |
+./spec/experiment_result_worker_spec.rb[1:3:2:2] | passed | 0.0025 seconds  |
+./spec/util_spec.rb[1:1:1]                       | passed | 0.00021 seconds |
+./spec/util_spec.rb[1:2:1]                       | passed | 0.00016 seconds |
+./spec/util_spec.rb[1:3:1]                       | passed | 0.00014 seconds |
+./spec/util_spec.rb[1:3:2]                       | passed | 0.00016 seconds |
+./spec/util_spec.rb[1:3:3]                       | passed | 0.00012 seconds |
+./spec/util_spec.rb[1:3:4]                       | passed | 0.00011 seconds |

data/spec/experiment_control_spec.rb

@@ -0,0 +1,60 @@
+require "spec_helper"
+require "async_experiments/experiment_control"
+require "async_experiments/experiment_result_worker"
+
+RSpec.describe AsyncExperiments::ExperimentControl do
+  let(:name) { :some_experiment }
+  let(:id) { double(:id) }
+
+  let(:candidate_args) { [1, 2, 3] }
+  let(:candidate_worker) { double(:candidate_worker, perform_async: nil) }
+  let(:candidate_config) {
+    {
+      worker: candidate_worker,
+      args: candidate_args,
+    }
+  }
+
+  let(:run_output) { [{some: "output"}] }
+
+  class TestClass
+    include AsyncExperiments::ExperimentControl
+
+    def call(name, run_output, candidate_config)
+      experiment_control(name, candidate: candidate_config) do
+        Timecop.travel(Time.now + 1.5)
+        run_output
+      end
+    end
+  end
+
+  subject { TestClass.new }
+
+  before do
+    allow(SecureRandom).to receive(:uuid).and_return(id)
+  end
+
+  describe "#experiment_control(name, candidate: candidate_config)" do
+    it "triggers an ExperimentResultWorker with the control output and duration" do
+      expect(AsyncExperiments::ExperimentResultWorker).to receive(:perform_async)
+        .with(name, id, run_output, instance_of(Float), :control)
+
+      subject.call(name, run_output, candidate_config)
+    end
+
+    it "triggers the candidate worker with its arguments and the experiment config" do
+      expect(candidate_worker).to receive(:perform_async)
+        .with(*candidate_args,
+          name: name,
+          id: id,
+        )
+
+      subject.call(name, run_output, candidate_config)
+    end
+
+    it "returns the control run output" do
+      output = subject.call(name, run_output, candidate_config)
+      expect(output).to eq(run_output)
+    end
+  end
+end

data/spec/experiment_error_worker_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+require "async_experiments"
+require "async_experiments/experiment_error_worker"
+
+RSpec.describe AsyncExperiments::ExperimentErrorWorker do
+  let(:name) { "some_experiment" }
+  let(:error) { "Something went wrong" }
+
+  let(:statsd) { double(:statsd, increment: nil) }
+  let(:redis) { double(:redis, rpush: nil) }
+
+  subject { described_class.new }
+
+  before do
+    allow(Sidekiq).to receive(:redis).and_yield(redis)
+    AsyncExperiments.statsd = statsd
+  end
+
+  it "uses the 'experiments' queue" do
+    Sidekiq::Testing.fake! do
+      described_class.perform_async
+
+      expect(described_class.jobs.size).to eq(1)
+    end
+  end
+
+  it "increments the statsd error count for the experiment" do
+    expect(statsd).to receive(:increment).with("experiments.#{name}.exceptions")
+    subject.perform(name, error)
+  end
+
+  it "stores the exception for later reporting" do
+    expect(redis).to receive(:rpush).with("experiments:#{name}:exceptions", error)
+    subject.perform(name, error)
+  end
+end

data/spec/experiment_result_spec.rb

@@ -0,0 +1,136 @@
+require "spec_helper"
+require "async_experiments/experiment_result"
+
+RSpec.describe AsyncExperiments::ExperimentResult do
+  let(:id) { SecureRandom.uuid }
+  let(:name) { :test_experiment }
+
+  let(:control_run_output) { "control output" }
+  let(:control_duration) { 10.0 }
+
+  let(:candidate_run_output) { "candidate output" }
+  let(:candidate_duration) { 5.0 }
+
+  let(:redis) { double(:redis, del: nil) }
+  let(:statsd) { double(:statsd, timing: nil, increment: nil) }
+
+  let(:control) { described_class.new(name, id, :control, redis, statsd, control_run_output, control_duration) }
+  let(:candidate) { described_class.new(name, id, :candidate, redis, statsd, candidate_run_output, candidate_duration) }
+
+  describe "#key" do
+    it "builds a key from the name and ID" do
+      expect(control.key).to eq("test_experiment:#{id}")
+    end
+  end
+
+  describe "#store_run_output" do
+    it "stores the branch's run output and duration" do
+      expect(redis).to receive(:set)
+        .with("experiments:#{name}:#{id}:candidate", {
+          run_output: candidate_run_output,
+          duration: candidate_duration,
+        }.to_json)
+
+      candidate.store_run_output
+    end
+  end
+
+  describe "control#process_run_output(candidate)" do
+    before do
+      allow(redis).to receive(:rpush)
+    end
+
+    it "reports the control and candidate durations to statsd" do
+      expect(statsd).to receive(:timing)
+        .with("experiments.#{name}.control", control_duration)
+
+      expect(statsd).to receive(:timing)
+        .with("experiments.#{name}.candidate", candidate_duration)
+
+      control.process_run_output(candidate)
+    end
+
+    it "deletes the candidate data from redis" do
+      expect(redis).to receive(:del).with("experiments:#{name}:#{id}:candidate")
+
+      control.process_run_output(candidate)
+    end
+
+    context "if there's variation between the outputs" do
+      it "increments the mismatch count in statsd" do
+        expect(statsd).to receive(:increment)
+          .with("experiments.#{name}.mismatches")
+
+        control.process_run_output(candidate)
+      end
+
+      it "logs the mismatch to redis" do
+        expect(redis).to receive(:rpush).with(
+          "experiments:#{name}:mismatches",
+          [["~", "", "control output", "candidate output"]].to_json,
+        )
+
+        control.process_run_output(candidate)
+      end
+    end
+
+    context "if there's no variation" do
+      let(:candidate_run_output) { control_run_output }
+
+      it "does not increment the mismatch count" do
+        expect(statsd).not_to receive(:increment)
+        control.process_run_output(candidate)
+      end
+
+      it "does not log the mismatch to redis" do
+        expect(redis).not_to receive(:rpush)
+        control.process_run_output(candidate)
+      end
+    end
+  end
+
+  describe ".new" do
+    let(:type) { :candidate }
+
+    context "if duration and output are provided" do
+      it "uses those" do
+        candidate = described_class.new(name, id, type, redis, statsd, "arbitrary output", 1.23)
+        expect(candidate.run_output).to eq("arbitrary output")
+        expect(candidate.duration).to eq(1.23)
+      end
+    end
+
+    context "if duration and output are not provided" do
+      context "and redis has the data" do
+        before do
+          allow(redis).to receive(:get).with("experiments:#{name}:#{id}:#{type}").and_return({
+            run_output: candidate_run_output,
+            duration: candidate_duration,
+          }.to_json)
+        end
+
+        it "uses the redis data" do
+          candidate = described_class.new(name, id, type, redis, statsd)
+          expect(candidate.run_output).to eq(candidate_run_output)
+          expect(candidate.duration).to eq(candidate_duration)
+        end
+
+        it "is considered available" do
+          candidate = described_class.new(name, id, type, redis, statsd)
+          expect(candidate.available?).to eq(true)
+        end
+      end
+
+      context "but redis does not have the data" do
+        before do
+          allow(redis).to receive(:get).and_return("")
+        end
+
+        it "is considered unavailable" do
+          missing_candidate = described_class.new(name, id, type, redis, statsd)
+          expect(missing_candidate.available?).to eq(false)
+        end
+      end
+    end
+  end
+end

data/spec/experiment_result_worker_spec.rb

@@ -0,0 +1,106 @@
+require "spec_helper"
+require "async_experiments"
+require "async_experiments/experiment_result"
+require "async_experiments/experiment_result_worker"
+
+RSpec.describe AsyncExperiments::ExperimentResultWorker do
+  let(:name) { "some_experiment" }
+  let(:id) { SecureRandom.uuid }
+
+  let(:statsd) { double(:statsd) }
+  let(:redis) { double(:redis) }
+  let(:redis_key) { double(:redis_key) }
+
+  let(:control_output) { "control output" }
+  let(:control_duration) { 10.0 }
+
+  let(:candidate_output) { "candidate output" }
+  let(:candidate_duration) { 5.0 }
+
+  let(:control) {
+    double(:control,
+      control?: true,
+      candidate?: false,
+      key: redis_key,
+    )
+  }
+  let(:candidate) {
+    double(:candidate,
+      candidate?: true,
+      control?: false,
+      key: redis_key,
+    )
+  }
+
+  subject { described_class.new }
+
+  before do
+    allow(Sidekiq).to receive(:redis).and_yield(redis)
+    AsyncExperiments.statsd = statsd
+  end
+
+  it "uses the 'experiments' queue" do
+    Sidekiq::Testing.fake! do
+      described_class.perform_async
+
+      expect(described_class.jobs.size).to eq(1)
+    end
+  end
+
+  context "if we're the candidate" do
+    let(:type) { "candidate" }
+
+    it "stores the run output and duration" do
+      expect(AsyncExperiments::ExperimentResult).to receive(:new)
+        .with(name, id, type.to_sym, redis, statsd, candidate_output, candidate_duration)
+        .and_return(candidate)
+
+      expect(candidate).to receive(:store_run_output)
+
+      subject.perform(name, id, candidate_output, candidate_duration, type)
+    end
+  end
+
+  context "if we're the control" do
+    let(:type) { "control" }
+
+    before do
+      expect(AsyncExperiments::ExperimentResult).to receive(:new)
+        .with(name, id, :control, redis, statsd, control_output, control_duration)
+        .and_return(control)
+
+      expect(AsyncExperiments::ExperimentResult).to receive(:new)
+        .with(name, id, :candidate, redis, statsd)
+        .and_return(candidate)
+    end
+
+    context "and the candidate is available" do
+      before do
+        allow(candidate).to receive(:available?).and_return(true)
+      end
+
+      it "processes the run output with the candidate" do
+        expect(control).to receive(:process_run_output).with(candidate)
+        subject.perform(name, id, control_output, control_duration, type)
+      end
+    end
+
+    context "but the candidate is unavailable" do
+      before do
+        allow(candidate).to receive(:available?).and_return(false)
+        allow(described_class).to receive(:perform_in)
+      end
+
+      it "does not process the run output" do
+        expect(control).not_to receive(:process_run_output)
+        subject.perform(name, id, control_output, control_duration, type)
+      end
+
+      it "schedules the job to run again later" do
+        args = [name, id, control_output, control_duration]
+        expect(described_class).to receive(:perform_in).with(5, *args, type.to_sym)
+        subject.perform(*args, type)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,101 @@
+require "byebug"
+require "timecop"
+require 'sidekiq/testing'
+Sidekiq::Logging.logger = nil
+
+ENV["RAISE_EXPERIMENT_ERRORS"] = "1"
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # The settings below are suggested to provide a good initial experience
+  # with RSpec, but feel free to customize to your heart's content.
+
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  config.before(:each) do
+    Sidekiq::Worker.clear_all
+  end
+end

data/spec/util_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+require "async_experiments/util"
+
+RSpec.describe AsyncExperiments::Util do
+  describe ".present?(object)" do
+    it "returns true unless empty or nil" do
+      expect(described_class.present?("This is a test")).to eq(true)
+      expect(described_class.present?([1, 2])).to eq(true)
+      expect(described_class.present?({a: 1})).to eq(true)
+
+      expect(described_class.present?(nil)).to eq(false)
+      expect(described_class.present?("")).to eq(false)
+      expect(described_class.present?([])).to eq(false)
+      expect(described_class.present?({})).to eq(false)
+    end
+  end
+
+  describe ".blank?(object)" do
+    it "returns true if empty or nil" do
+      expect(described_class.blank?(nil)).to eq(true)
+      expect(described_class.blank?("")).to eq(true)
+      expect(described_class.blank?([])).to eq(true)
+      expect(described_class.blank?({})).to eq(true)
+
+      expect(described_class.blank?("This is a test")).to eq(false)
+      expect(described_class.blank?([1, 2])).to eq(false)
+      expect(described_class.blank?({a: 1})).to eq(false)
+      expect(described_class.blank?(5)).to eq(false)
+      expect(described_class.blank?(5.5)).to eq(false)
+    end
+  end
+
+  describe ".deep_symbolize_keys(hash)" do
+    it "changes top level string keys to symbols" do
+      symbolized_hash = described_class.deep_symbolize_keys(
+        "test" => 1,
+      )
+
+      expect(symbolized_hash).to eq(
+        test: 1,
+      )
+    end
+
+    it "changes lower level string keys to symbols" do
+      symbolized_hash = described_class.deep_symbolize_keys(
+        test: {
+          "test" => 1,
+        },
+      )
+
+      expect(symbolized_hash).to eq(
+        test: {
+          test: 1,
+        },
+      )
+    end
+
+    it "leaves existing symbol keys alone" do
+      symbolized_hash = described_class.deep_symbolize_keys(
+        test: 1,
+      )
+
+      expect(symbolized_hash).to eq(
+        test: 1,
+      )
+    end
+
+    it "leaves other object keys alone" do
+      symbolized_hash = described_class.deep_symbolize_keys(
+        1 => "test",
+      )
+
+      expect(symbolized_hash).to eq(
+        1 => "test",
+      )
+    end
+  end
+end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: async_experiments
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Elliot Crosby-McCullough
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: gem_publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- elliot.cm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENCE.txt
+- README.md
+- lib/async_experiments.rb
+- lib/async_experiments/candidate_worker.rb
+- lib/async_experiments/experiment_control.rb
+- lib/async_experiments/experiment_error_worker.rb
+- lib/async_experiments/experiment_result.rb
+- lib/async_experiments/experiment_result_worker.rb
+- lib/async_experiments/util.rb
+- lib/async_experiments/version.rb
+- spec/async_experiments_spec.rb
+- spec/candidate_worker_spec.rb
+- spec/examples.txt
+- spec/experiment_control_spec.rb
+- spec/experiment_error_worker_spec.rb
+- spec/experiment_result_spec.rb
+- spec/experiment_result_worker_spec.rb
+- spec/spec_helper.rb
+- spec/util_spec.rb
+homepage: http://github.com/alphagov/async_experiments
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: An asynchronous experiment framework.
+test_files:
+- spec/experiment_result_spec.rb
+- spec/spec_helper.rb
+- spec/experiment_error_worker_spec.rb
+- spec/experiment_result_worker_spec.rb
+- spec/async_experiments_spec.rb
+- spec/util_spec.rb
+- spec/experiment_control_spec.rb
+- spec/candidate_worker_spec.rb
+- spec/examples.txt