laboratory 0.1.1 → 0.1.2

data/Gemfile

@@ -1,6 +1,6 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in laboratory.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem 'rake', '~> 12.0'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    laboratory (0.1.1)
+    laboratory (0.1.2)
       redis (>= 2.1)
       sinatra (>= 1.2.6)
 

data/README.md

@@ -13,6 +13,18 @@ Laboratory is an A/B testing and Feature Flag framework for Rails. It's focused
 
 Laboratory builds upon great work from other gems, in particular [Split](https://github.com/splitrb/split).
 
+Laboratory is in active development, see the bottom for a todo list.
+
+## Preview of UI Interface
+
+### Viewing and Analysing experiments
+
+![ui-interface](https://imgur.com/l23iiet.png)
+
+### Editting an experiment
+
+![edit-ui-interface](https://imgur.com/O6JkDk0.png)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -126,6 +138,29 @@ experiment.record_event!('completed')
 
 Note the `#record_event!` method also takes an optional user parameter should you want to define the user specifically in this case. It defaults to a user with the current_user_id defined in the Laboratory configuration.
 
+### Temporarily overriding experiment variants
+
+Sometimes, when QA'ing or developing an experiment, you'll want to easily switch between variants without having to jump into the console. This can be managed via a url parameter by adding the following snippet to your application controller (this example is for Rails, but a similar approach would work for other frameworks):
+
+```ruby
+
+around_action :override_laboratory_experiments!
+
+def override_laboratory_experiments!
+  Laboratory::Experiment.override!(params[:exp])
+  yield
+  Laboratory::Experiment.clear_overrides!
+end
+```
+
+This then allows you navigate to a urls like:
+
+http://yourwebsite.com?exp[blue_button_ab_test]=variant_a
+
+and
+
+http://yourwebsite.com?exp[blue_button_ab_test]=control
+
 ### Using the Laboratory UI
 
 It's easy to analyse and manage your experiment from the dashboard. In routes.rb, mount the dashboard behind your appropriate authentication layer (this example uses Devise):
@@ -208,8 +243,6 @@ user.variant_for_experiment(experiment) # Note: This returns nil if the user is
 
 **Updating an experiment's variants**:
 
-Note: This would wipe all users from the experiment.
-
 ```ruby
 experiment = Laboratory::Experiment.find('blue_button_cta')
 control = experiment.variants.find { |variant| variant.id == 'control' }
@@ -252,6 +285,11 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+### Todo List
+
+- [ ] Test in a multi-threaded puma environment
+- [ ] Test performance in a A/A test on production
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/butternutbox/laboratory. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to that.

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,14 +1,14 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "laboratory"
+require 'bundler/setup'
+require 'laboratory'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
+# require 'pry'
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/laboratory.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.email = ['niall@butternutbox.com']
 
   spec.summary = 'Laboratory: An A/B Testing and Feature Flag system for Ruby'
-  spec.description = 'Laboratory: An A/B Testing and Feature Flag system for Ruby'
+  spec.description = 'An A/B Testing and Feature Flag system for Ruby'
   spec.homepage = 'https://github.com/butternutbox/laboratory'
   spec.license = 'MIT'
   spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
@@ -17,9 +17,12 @@ Gem::Specification.new do |spec|
   spec.metadata['changelog_uri'] = 'https://github.com/butternutbox/laboratory/releases'
 
   # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added
+  # into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
   spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -28,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'redis', '>= 2.1'
   spec.add_dependency 'sinatra', '>= 1.2.6'
 
-  spec.add_development_dependency 'rack-test',   '~> 1.1'
-  spec.add_development_dependency 'rspec', '~> 3.8'
   spec.add_development_dependency 'fakeredis', '~> 0.8'
+  spec.add_development_dependency 'rack-test', '~> 1.1'
+  spec.add_development_dependency 'rspec', '~> 3.8'
 end

data/lib/laboratory/adapters/redis_adapter.rb

@@ -1,23 +1,26 @@
 module Laboratory
   module Adapters
-    class RedisAdapter
+    class RedisAdapter # rubocop:disable Metrics/ClassLength
       attr_reader :redis
 
-      ALL_EXPERIMENTS_KEYS_KEY = 'SWITCH_ALL_EXPERIMENT_KEYS'.freeze
+      ALL_EXPERIMENTS_KEYS_KEY = 'LABORATORY_ALL_EXPERIMENT_KEYS'.freeze
 
       def initialize(url:)
         @redis = Redis.new(url: url)
-
-        if !redis.get(ALL_EXPERIMENTS_KEYS_KEY)
-          redis.set(ALL_EXPERIMENTS_KEYS_KEY, [])
-        end
       end
 
       def write(experiment)
-        redis.set(redis_key(experiment_id: experiment.id), experiment_to_json(experiment))
-        # Write to ALL_EXPERIMENTS_KEY_KEY if it isn't already there.
-        experiment_ids = JSON.parse(redis.get(ALL_EXPERIMENTS_KEYS_KEY))
-        experiment_ids << experiment.id unless experiment_ids.include?(experiment.id)
+        key = redis_key(experiment_id: experiment.id)
+        json = experiment_to_json(experiment)
+        redis.set(key, json)
+
+        # Write to ALL_EXPERIMENTS_KEYS_KEY if it isn't already there.
+        experiment_ids = fetch_all_experiment_ids
+
+        unless experiment_ids.include?(experiment.id)
+          experiment_ids << experiment.id
+        end
+
         redis.set(ALL_EXPERIMENTS_KEYS_KEY, experiment_ids.to_json)
       end
 
@@ -26,11 +29,12 @@ module Laboratory
         response = redis.get(key)
 
         return nil if response.nil?
+
         parse_json_to_experiment(JSON.parse(response))
       end
 
       def read_all
-        experiment_ids = JSON.parse(redis.get(ALL_EXPERIMENTS_KEYS_KEY))
+        experiment_ids = fetch_all_experiment_ids
         experiment_ids.map do |experiment_id|
           read(experiment_id)
         end
@@ -41,18 +45,27 @@ module Laboratory
         redis.del(key)
 
         # Remove from ALL_EXPERIMENTS_KEY_KEY
-        experiment_ids = JSON.parse(redis.get(ALL_EXPERIMENTS_KEYS_KEY))
+        experiment_ids = fetch_all_experiment_ids
         experiment_ids.delete(experiment_id)
         redis.set(ALL_EXPERIMENTS_KEYS_KEY, experiment_ids.to_json)
       end
 
       def delete_all
-        experiment_ids = JSON.parse(redis.get(ALL_EXPERIMENTS_KEYS_KEY))
+        experiment_ids = fetch_all_experiment_ids
         experiment_ids.each { |experiment_id| delete(experiment_id) }
       end
 
       private
 
+      def fetch_all_experiment_ids
+        response = redis.get(ALL_EXPERIMENTS_KEYS_KEY)
+        if response
+          JSON.parse(response)
+        else
+          []
+        end
+      end
+
       def redis_key(experiment_id:)
         "laboratory_#{experiment_id}"
       end
@@ -79,9 +92,11 @@ module Laboratory
 
       def experiment_events_to_hash(events)
         events.map do |event|
+          event_recordings =
+            experiment_event_recordings_to_hash(event.event_recordings)
           {
             id: event.id,
-            event_recordings: experiment_event_recordings_to_hash(event.event_recordings)
+            event_recordings: event_recordings
           }
         end
       end
@@ -127,9 +142,13 @@ module Laboratory
 
       def parse_json_to_experiment_events(events_json)
         events_json.map do |json|
+          event_recordings = parse_json_to_experiment_event_recordings(
+            json['event_recordings']
+          )
+
           Experiment::Event.new(
             id: json['id'],
-            event_recordings: parse_json_to_experiment_event_recordings(json['event_recordings'])
+            event_recordings: event_recordings
           )
         end
       end

data/lib/laboratory/algorithms/random.rb

@@ -2,7 +2,7 @@ module Laboratory
   module Algorithms
     class Random
       def self.pick!(variants)
-        variants.sort_by { |variant| - variant.percentage * rand() }.first
+        variants.min_by { |variant| - variant.percentage * rand }
       end
 
       def self.id

data/lib/laboratory/calculations/confidence_level.rb

@@ -1,7 +1,7 @@
 module Laboratory
   module Calculations
     module ConfidenceLevel
-      def self.calculate(n1:, p1:, n2:, p2:)
+      def self.calculate(n1:, p1:, n2:, p2:) # rubocop:disable Naming/MethodParameterName
         cvr1 = p1.fdiv(n1)
         cvr2 = p2.fdiv(n2)
 
@@ -15,7 +15,7 @@ module Laboratory
         percentage_from_z_score(-z).round(4)
       end
 
-      def self.percentage_from_z_score(z)
+      def self.percentage_from_z_score(z) # rubocop:disable Naming/MethodParameterName, Metrics/AbcSize, Metrics/MethodLength
         return 0 if z < -6.5
         return 1 if z > 6.5
 
@@ -23,10 +23,13 @@ module Laboratory
         sum = 0
         term = 1
         k = 0
+        const = 0.3989422804
 
         loop_stop = Math.exp(-23)
-        while term.abs > loop_stop do
-          term = 0.3989422804 * ((-1)**k) * (z**k) / (2 * k + 1) / (2**k) * (z**(k + 1)) / factk
+        while term.abs > loop_stop do # rubocop:disable Style/WhileUntilDo
+          term =
+            const * ((-1)**k) * (z**k) / (2 * k + 1) / (2**k) * (z**(k + 1)) / factk # rubocop:disable Layout/LineLength
+
           sum += term
           k += 1
           factk *= k

data/lib/laboratory/calculations/z_score.rb

@@ -6,33 +6,35 @@ module Laboratory
       # n: Total population
       # p: conversion percentage
 
-      def self.calculate(n1:, p1:, n2:, p2:)
-        p_1 = p1.to_f
-        p_2 = p2.to_f
+      def self.calculate(n1:, p1:, n2:, p2:) # rubocop:disable Metrics/AbcSize, Naming/MethodParameterName, Metrics/MethodLength
+        p1_float = p1.to_f
+        p2_float = p2.to_f
 
-        n_1 = n1.to_f
-        n_2 = n2.to_f
+        n1_float = n1.to_f
+        n2_float = n2.to_f
 
         # Formula for standard error: root(pq/n) = root(p(1-p)/n)
-        s_1 = Math.sqrt(p_1 * (1 - p_1) / n_1)
-        s_2 = Math.sqrt(p_2 * (1 - p_2) / n_2)
+        s1_float = Math.sqrt(p1_float * (1 - p1_float) / n1_float)
+        s2_float = Math.sqrt(p2_float * (1 - p2_float) / n2_float)
 
-        # Formula for pooled error of the difference of the means: root(π*(1-π)*(1/na+1/nc)
-        # π = (xa + xc) / (na + nc)
-        pi = (p_2 * n_2 + p_1 * n_1) / (n_2 + n_1)
-        s_p = Math.sqrt(pi * (1 - pi) * (1 / n_2 + 1 / n_1))
+        # Formula for pooled error of the difference of the means:
+        # root(pi*(1-pi)*(1/na+1/nc)
+        # pi = (xa + xc) / (na + nc)
+        pi = (p2_float * n2_float + p1_float * n1_float) / (n2_float + n1_float)
+        s_p = Math.sqrt(pi * (1 - pi) * (1 / n2_float + 1 / n1_float))
 
-        # Formula for unpooled error of the difference of the means: root(sa**2/na + sc**2/nc)
-        s_unp = Math.sqrt(s_2**2 + s_1**2)
+        # Formula for unpooled error of the difference of the means:
+        # root(sa**2/pi*a + sc**2/nc)
+        s_unp = Math.sqrt(s2_float**2 + s1_float**2)
 
         # Boolean variable decides whether we can pool our variances
-        pooled = s_2 / s_1 < 2 && s_1 / s_2 < 2
+        pooled = s2_float / s1_float < 2 && s1_float / s2_float < 2
 
         # Assign standard error either the pooled or unpooled variance
         se = pooled ? s_p : s_unp
 
         # Calculate z-score
-        z_score = (p_2 - p_1) / (se)
+        z_score = (p2_float - p1_float) / se
 
         z_score.round(4)
       end

data/lib/laboratory/experiment/analysis_summary.rb

@@ -17,8 +17,9 @@ module Laboratory
       end
 
       def performance_delta_between_highest_and_lowest
-        numerator = (conversion_rate_for_variant(highest_performing_variant) -
-          conversion_rate_for_variant(lowest_performing_variant))
+        numerator =
+          (conversion_rate_for_variant(highest_performing_variant) -
+            conversion_rate_for_variant(lowest_performing_variant))
         denominator = conversion_rate_for_variant(lowest_performing_variant)
         numerator.fdiv(denominator).round(2)
       end
@@ -57,9 +58,10 @@ module Laboratory
       end
 
       def sorted_variants
-        relevant_variants.sort_by { |variant|
-          conversion_rate_for_variant(variant)
-        }.reverse
+        relevant_variants.sort_by do |variant|
+          # Order in descending order
+          -1 * conversion_rate_for_variant(variant)
+        end
       end
 
       def event_for_variant(variant)

data/lib/laboratory/experiment/variant.rb

@@ -26,7 +26,11 @@ module Laboratory
       def changeset
         set = {}
         set[:id] = [_original_id, id] if _original_id != id
-        set[:percentage] = [_original_percentage, percentage] if _original_percentage != percentage
+
+        if _original_percentage != percentage
+          set[:percentage] = [_original_percentage, percentage]
+        end
+
         set
       end
     end

data/lib/laboratory/experiment.rb

@@ -1,5 +1,19 @@
 module Laboratory
-  class Experiment
+  class Experiment # rubocop:disable Metrics/ClassLength
+    class << self
+      def overrides
+        @overrides || {}
+      end
+
+      def override!(overrides)
+        @overrides = overrides
+      end
+
+      def clear_overrides!
+        @overrides = {}
+      end
+    end
+
     class UserNotInExperimentError < StandardError; end
     class ClashingExperimentIdError < StandardError; end
     class MissingExperimentIdError < StandardError; end
@@ -15,7 +29,7 @@ module Laboratory
       :changelog
     )
 
-    def initialize(id:, variants:, algorithm: Algorithms::Random, changelog: [])
+    def initialize(id:, variants:, algorithm: Algorithms::Random, changelog: []) # rubocop:disable Metrics/MethodLength
       @id = id
       @algorithm = algorithm
       @changelog = changelog
@@ -24,7 +38,7 @@ module Laboratory
       # This also helps when decoding from adapters
 
       @variants =
-        if variants.all? { |variant| variant.instance_of?(Laboratory::Experiment::Variant) }
+        if variants.all? { |variant| variant.instance_of?(Experiment::Variant) }
           variants
         elsif variants.all? { |variant| variant.instance_of?(Hash) }
           variants.map do |variant|
@@ -83,9 +97,15 @@ module Laboratory
       save
     end
 
-    def variant(user: Laboratory.config.current_user)
-      selected_variant = variants.find { |variant| variant.participant_ids.include?(user.id)}
-      return selected_variant if !selected_variant.nil?
+    def variant(user: Laboratory.config.current_user) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      return variant_overridden_with if overridden?
+
+      selected_variant =
+        variants.find do |variant|
+          variant.participant_ids.include?(user.id)
+        end
+
+      return selected_variant unless selected_variant.nil?
 
       variant = algorithm.pick!(variants)
       variant.add_participant(user)
@@ -110,7 +130,7 @@ module Laboratory
       variant
     end
 
-    def record_event!(event_id, user: Laboratory.config.current_user)
+    def record_event!(event_id, user: Laboratory.config.current_user) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       variant = variants.find { |s| s.participant_ids.include?(user.id) }
       raise UserNotInExperimentError unless variant
 
@@ -139,6 +159,7 @@ module Laboratory
 
     def save
       raise errors.first unless valid?
+
       unless changeset.empty?
         changelog_item = Laboratory::Experiment::ChangelogItem.new(
           changes: changeset,
@@ -151,7 +172,7 @@ module Laboratory
       Laboratory.config.adapter.write(self)
     end
 
-    def valid?
+    def valid? # rubocop:disable Metrics/AbcSize
       valid_variants =
         variants.all? do |variant|
           !variant.id.nil? && !variant.percentage.nil?
@@ -165,14 +186,27 @@ module Laboratory
 
     private
 
-    def changeset
+    def overridden?
+      self.class.overrides.key?(id) && !variant_overridden_with.nil?
+    end
+
+    def variant_overridden_with
+      variants.find { |v| v.id == self.class.overrides[id] }
+    end
+
+    def changeset # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       set = {}
       set[:id] = [_original_id, id] if _original_id != id
-      set[:algorithm] = [_original_algorithm, algorithm] if _original_algorithm != algorithm
 
-      variants_changeset = variants.map { |variant|
-        { variant.id => variant.changeset }
-      }
+      if _original_algorithm != algorithm
+        set[:algorithm] = [_original_algorithm, algorithm]
+      end
+
+      variants_changeset =
+        variants.map do |variant|
+          { variant.id => variant.changeset }
+        end
+
       variants_changeset.reject! do |change|
         change.values.all?(&:empty?)
       end
@@ -181,23 +215,25 @@ module Laboratory
       set
     end
 
-    def errors
+    def errors # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       errors = []
 
-      missing_variant_ids = variants.any? do |variant|
-        variant.id.nil?
-      end
+      missing_variant_ids =
+        variants.any? do |variant|
+          variant.id.nil?
+        end
 
-      missing_variant_percentages = variants.any? do |variant|
-        variant.percentage.nil?
-      end
+      missing_variant_percentages =
+        variants.any? do |variant|
+          variant.percentage.nil?
+        end
 
       incorrect_percentage_total = variants.map(&:percentage).sum != 100
 
       errors << MissingExperimentIdError if id.nil?
       errors << MissingExperimentAlgorithmError if algorithm.nil?
       errors << MissingExperimentVariantIdError if missing_variant_ids
-      errors << MissingExperimentVariantPercentageError if missing_variant_percentages
+      errors << MissingExperimentVariantPercentageError if missing_variant_percentages # rubocop:disable Layout/LineLength
       errors << IncorrectPercentageTotalError if incorrect_percentage_total
 
       errors

data/lib/laboratory/ui/helpers.rb

@@ -1,7 +1,7 @@
 module Laboratory
   module UIHelpers
     def url(*path_parts)
-      [ path_prefix, path_parts ].join("/").squeeze('/')
+      [path_prefix, path_parts].join('/').squeeze('/')
     end
 
     def path_prefix
@@ -9,26 +9,31 @@ module Laboratory
     end
 
     def experiment_url(experiment)
-      url('experiments', experiment.id, 'edit')
+      url('experiments', CGI.escape(experiment.id), 'edit')
     end
 
     def update_percentages_url(experiment)
-      url('experiments', experiment.id, 'update_percentages')
+      url('experiments', CGI.escape(experiment.id), 'update_percentages')
     end
 
     def assign_users_to_variant_url(experiment)
-      url('experiments', experiment.id, 'assign_users')
+      url('experiments', CGI.escape(experiment.id), 'assign_users')
     end
 
     def reset_experiment_url(experiment)
-      url('experiments', experiment.id, 'reset')
+      url('experiments', CGI.escape(experiment.id), 'reset')
     end
 
     def analysis_summary(experiment, event_id)
+      return if experiment.variants.length < 2
+
       analysis = experiment.analysis_summary_for(event_id)
-      return if analysis.highest_performing_variant == analysis.lowest_performing_variant
 
-      "#{analysis.highest_performing_variant.id} is performing #{analysis.performance_delta_between_highest_and_lowest * 100}% better than #{analysis.lowest_performing_variant.id}. I'm #{analysis.confidence_level_in_performance_delta * 100}% certain of this."
+      "#{analysis.highest_performing_variant.id} is performing" \
+      " #{analysis.performance_delta_between_highest_and_lowest * 100}%" \
+      " better than #{analysis.lowest_performing_variant.id}. I'm" \
+      " #{analysis.confidence_level_in_performance_delta * 100}% certain of" \
+      ' this.'
     end
   end
 end

data/lib/laboratory/ui.rb

@@ -19,13 +19,13 @@ module Laboratory
     end
 
     get '/experiments/:id/edit' do
-      @experiment = Laboratory::Experiment.find(params[:id])
+      @experiment = Laboratory::Experiment.find(CGI.unescape(params[:id]))
       erb :edit
     end
 
     # params = {variants: { control => 40, variant_a => 60 }}
     post '/experiments/:id/update_percentages' do
-      experiment = Laboratory::Experiment.find(params[:id])
+      experiment = Laboratory::Experiment.find(CGI.unescape(params[:id]))
 
       params[:variants].each do |variant_id, percentage|
         variant = experiment.variants.find { |v| v.id == variant_id }
@@ -38,7 +38,7 @@ module Laboratory
 
     # params = {variant_id: 'control', user_ids: []}
     post '/experiments/:id/assign_users' do
-      experiment = Laboratory::Experiment.find(params[:id])
+      experiment = Laboratory::Experiment.find(CGI.unescape(params[:id]))
       variant = experiment.variants.find { |v| v.id == params[:variant_id] }
       user_ids = params[:user_ids].split("\r\n")
 
@@ -51,7 +51,7 @@ module Laboratory
     end
 
     post '/experiments/:id/reset' do
-      experiment = Laboratory::Experiment.find(params[:id])
+      experiment = Laboratory::Experiment.find(CGI.unescape(params[:id]))
       experiment.reset
       redirect experiment_url(experiment)
     end

data/lib/laboratory/version.rb

@@ -1,3 +1,3 @@
 module Laboratory
-  VERSION = "0.1.1"
+  VERSION = '0.1.2'.freeze
 end