field_test 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21a7309f61348d5b03aa7a6073e2d7493dbc4f9a
-  data.tar.gz: 486affca9a9ade457fa2481925995bdc55b4a954
+  metadata.gz: 177ccc428012c8862f9d798b6cf58629d41d54bb
+  data.tar.gz: 5b57269ef51ea55f5f10ff280d3c8aac895db01d
 SHA512:
-  metadata.gz: b9c4d1fa745aa15f132cf8c74dcfd4f456864601c661139f96bef2b9cd17266b3a8c1dfcaec56d8b7fed21617e25d83c51ead8e73a7d7c59ae0ac057c5959eff
-  data.tar.gz: d971c7c1907456a6af2162738ad4473891b02c17fd4b9b48e316757b596291cc84ee6dee73016f573284113f73a0884f611e20ab2e053af99f4929ada7e0a361
+  metadata.gz: e014861b6912b47d5320511ebfbc0a5a8abd9470fc6a2f8b4c25f0c7d5cd07ff217836cb3d6f6bcda44a1d615fb3bf782ebdfb3161c49a292df251b581717ed4
+  data.tar.gz: d12c7f9d920bd6aa0df5c5aa15876b31c9693cca38f3fd628013ffa3f72e2323852405ea50d3fc9719509a78a8abfdaa2b4a4316e899fb37b692c7149acdfaf4

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1
+
+- Added basic web UI
+
 ## 0.1.0
 
 - First release

data/README.md

@@ -6,6 +6,8 @@
 - Seamlessly handles the transition from anonymous visitor to logged in user
 - Results are stored in your database
 
+Uses [Bayesian methods](http://www.evanmiller.org/bayesian-ab-testing.html) to evaluate results so you don’t need to choose a sample size ahead of time.
+
 ## Installation
 
 Add this line to your application’s Gemfile:
@@ -20,6 +22,14 @@ And run:
 rails g field_test:install
 ```
 
+And mount the dashboard in your `config/routes.rb`:
+
+```ruby
+mount FieldTest::Engine, at: "field_test"
+```
+
+Be sure to [secure the dashboard](#security) in production.
+
 ## Getting Started
 
 Add an experiment to `config/field_test.yml`.
@@ -28,9 +38,9 @@ Add an experiment to `config/field_test.yml`.
 experiments:
   button_color:
     variants:
-      - control
       - red
       - green
+      - blue
 ```
 
 Refer to it in views, controllers, and mailers.
@@ -76,18 +86,47 @@ For mailers, you need to specify a participant:
 field_test(:button_color, participant: "test@example.org")
 ```
 
+Keep track of when experiments started and ended.
+
+```yml
+experiments:
+  button_colors:
+    started_at: 2016-12-01 14:00:00
+    ended_at: 2016-12-08 14:00:00
+```
+
 ## Funnels
 
 For advanced funnels, we recommend an analytics platform like [Ahoy](https://github.com/ankane/ahoy) or [Mixpanel](https://mixpanel.com/).
 
 You can pass experiments and variants as properties.
 
+## Security
+
+#### Basic Authentication
+
+Set the following variables in your environment or an initializer.
+
+```ruby
+ENV["FIELD_TEST_USERNAME"] = "link"
+ENV["FIELD_TEST_PASSWORD"] = "hyrule"
+```
+
+#### Devise
+
+```ruby
+authenticate :user, -> (user) { user.admin? } do
+  mount FieldTest::Engine, at: "field_test"
+end
+```
+
+## Credits
+
+A huge thanks to [Evan Miller](http://www.evanmiller.org/) for deriving the Bayesian formulas.
+
 ## TODO
 
-- Add confidence to stats
-- Add [Bayesian confidence](http://www.evanmiller.org/bayesian-ab-testing.html) to results
 - Exclude bots
-- User interface
 
 ## History
 

data/app/controllers/field_test/home_controller.rb

@@ -0,0 +1,13 @@
+module FieldTest
+  class HomeController < ActionController::Base
+    layout "field_test/application"
+
+    protect_from_forgery
+
+    http_basic_authenticate_with name: ENV["FIELD_TEST_USERNAME"], password: ENV["FIELD_TEST_PASSWORD"] if ENV["FIELD_TEST_PASSWORD"]
+
+    def index
+      @active_experiments, @completed_experiments = FieldTest::Experiment.all.sort_by(&:id).partition { |e| e.active? }
+    end
+  end
+end

data/app/views/field_test/home/_experiments.html.erb

@@ -0,0 +1,42 @@
+<% experiments.each do |experiment| %>
+  <% results = experiment.results %>
+
+  <h2><%= experiment.name %></h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th style="width: 20%;">Participants</th>
+        <th style="width: 20%;">Conversions</th>
+        <th style="width: 20%;">Conversion Rate</th>
+        <th style="width: 20%;">Prob Winning</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% results.each do |variant, result| %>
+        <tr>
+          <td>
+            <%= variant %>
+            <% if variant == experiment.winner %>
+              <span style="color: #999;">winner</span>
+            <% end %>
+          </td>
+          <td><%= result[:participated] %></td>
+          <td><%= result[:converted] %></td>
+          <td>
+            <% if result[:conversion_rate] %>
+              <%= (100.0 * result[:conversion_rate]).round %>%
+            <% else %>
+              -
+            <% end %>
+          </td>
+          <td>
+            <% if result[:prob_winning] %>
+              <%= (100.0 * result[:prob_winning]).round %>%
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

data/app/views/field_test/home/index.html.erb

@@ -0,0 +1,11 @@
+<% if @active_experiments.any? %>
+  <h1>Active Experiments</h1>
+
+  <%= render partial: "experiments", locals: {experiments: @active_experiments} %>
+<% end %>
+
+<% if @completed_experiments.any? %>
+  <h1>Completed Experiments</h1>
+
+  <%= render partial: "experiments", locals: {experiments: @completed_experiments} %>
+<% end %>

data/app/views/layouts/field_test/application.html.erb

@@ -0,0 +1,39 @@
+<html>
+  <head>
+    <title>Field Test</title>
+
+    <style>
+      body {
+        margin: 0;
+        padding: 20px;
+        font-family: "Helvetica Neue", Arial, Helvetica, sans-serif;
+        font-size: 14px;
+        line-height: 1.4;
+        color: #333;
+      }
+
+      table {
+        width: 100%;
+        border-collapse: collapse;
+        border-spacing: 0;
+        margin-bottom: 20px;
+      }
+
+      th {
+        text-align: left;
+        border-bottom: solid 2px #ddd;
+      }
+
+      table td, table th {
+        padding: 8px;
+      }
+
+      td {
+        border-top: solid 1px #ddd;
+      }
+    </style>
+  </head>
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/config/routes.rb

@@ -0,0 +1,3 @@
+FieldTest::Engine.routes.draw do
+  root "home#index"
+end

data/field_test.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "railties"
   spec.add_dependency "activerecord"
+  spec.add_dependency "distribution"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"

data/lib/field_test.rb

@@ -1,11 +1,20 @@
+require "distribution/math_extension"
 require "field_test/experiment"
 require "field_test/engine"
 require "field_test/helpers"
+require "field_test/participant"
 require "field_test/version"
 
 module FieldTest
   class Error < StandardError; end
-  class ExperimentNotFound < Error; end
+class ExperimentNotFound < Error; end
+
+def self.config
+    # reload in dev
+    @config = nil if Rails.env.development?
+
+    @config ||= YAML.load(ERB.new(File.read("config/field_test.yml")).result)
+  end
 end
 
 ActiveSupport.on_load(:action_controller) do

data/lib/field_test/engine.rb

@@ -1,4 +1,8 @@
 module FieldTest
   class Engine < ::Rails::Engine
+    isolate_namespace FieldTest
+
+    # prevents conflict with field_test method in views
+    engine_name "field_test_engine"
   end
 end

data/lib/field_test/experiment.rb

@@ -1,19 +1,22 @@
 module FieldTest
   class Experiment
-    attr_reader :id, :variants, :winner
+    attr_reader :id, :name, :variants, :winner, :started_at, :ended_at
 
     def initialize(attributes)
       attributes = attributes.symbolize_keys
       @id = attributes[:id]
+      @name = attributes[:name] || @id.to_s.titleize
       @variants = attributes[:variants]
       @winner = attributes[:winner]
+      @started_at = Time.zone.parse(attributes[:started_at].to_s) if attributes[:started_at]
+      @ended_at = Time.zone.parse(attributes[:ended_at].to_s) if attributes[:ended_at]
     end
 
     def variant(participants, options = {})
       return winner if winner
       return variants.first if options[:exclude]
 
-      participants = standardize_participants(participants)
+      participants = FieldTest::Participant.standardize(participants)
       membership = membership_for(participants) || FieldTest::Membership.new(experiment: id)
 
       if options[:variant] && variants.include?(options[:variant])
@@ -37,7 +40,7 @@ module FieldTest
     end
 
     def convert(participants)
-      participants = standardize_participants(participants)
+      participants = FieldTest::Participant.standardize(participants)
       membership = membership_for(participants)
 
       if membership
@@ -54,7 +57,10 @@ module FieldTest
     end
 
     def results
-      data = memberships.group(:variant).group(:converted).count
+      data = memberships.group(:variant).group(:converted)
+      data = data.where("created_at >= ?", started_at) if started_at
+      data = data.where("created_at <= ?", ended_at) if ended_at
+      data = data.count
       results = {}
       variants.each do |variant|
         converted = data[[variant, true]].to_i
@@ -62,22 +68,51 @@ module FieldTest
         results[variant] = {
           participated: participated,
           converted: converted,
-          conversion_rate: converted.to_f / participated
+          conversion_rate: participated > 0 ? converted.to_f / participated : nil
         }
       end
+      case variants.size
+      when 1
+        results[variants[0]][:prob_winning] = 1
+      when 2, 3
+        variants.size.times do |i|
+          c = results.values[i]
+          b = results.values[(i + 1) % variants.size]
+          a = results.values[(i + 2) % variants.size]
+
+          alpha_a = 1 + a[:converted]
+          beta_a = 1 + a[:participated] - a[:converted]
+          alpha_b = 1 + b[:converted]
+          beta_b = 1 + b[:participated] - b[:converted]
+          alpha_c = 1 + c[:converted]
+          beta_c = 1 + c[:participated] - c[:converted]
+
+          results[variants[i]][:prob_winning] =
+            if variants.size == 2
+              prob_b_beats_a(alpha_b, beta_b, alpha_c, beta_c)
+            else
+              prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
+            end
+        end
+      end
       results
     end
 
-    def self.find(id)
-      # reload in dev
-      @config = nil if Rails.env.development?
+    def active?
+      !winner
+    end
 
-      @config ||= YAML.load(ERB.new(File.read("config/field_test.yml")).result)
+    def self.find(id)
+      experiment = all.index_by(&:id)[id.to_s]
+      raise FieldTest::ExperimentNotFound unless experiment
 
-      settings = @config["experiments"][id.to_s]
-      raise FieldTest::ExperimentNotFound unless settings
+      experiment
+    end
 
-      FieldTest::Experiment.new(settings.merge(id: id.to_s))
+    def self.all
+      FieldTest.config["experiments"].map do |id, settings|
+        FieldTest::Experiment.new(settings.merge(id: id.to_s))
+      end
     end
 
     private
@@ -87,8 +122,33 @@ module FieldTest
         participants.map { |part| memberships[part] }.compact.first
       end
 
-      def standardize_participants(participants)
-        Array(participants).map { |v| v.respond_to?(:model_name) ? "#{v.model_name.name}:#{v.id}" : v.to_s }
+      # formula from
+      # http://www.evanmiller.org/bayesian-ab-testing.html
+      def prob_b_beats_a(alpha_a, beta_a, alpha_b, beta_b)
+        total = 0.0
+
+        0.upto(alpha_b - 1) do |i|
+          total += Math.exp(Math.logbeta(alpha_a + i, beta_b + beta_a) -
+            Math.log(beta_b + i) - Math.logbeta(1 + i, beta_b) -
+            Math.logbeta(alpha_a, beta_a))
+        end
+
+        total
+      end
+
+      def prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
+        total = 0.0
+        0.upto(alpha_a - 1) do |i|
+          0.upto(alpha_b - 1) do |j|
+            total += Math.exp(Math.logbeta(alpha_c + i + j, beta_a + beta_b + beta_c) -
+              Math.log(beta_a + i) - Math.log(beta_b + j) -
+              Math.logbeta(1 + i, beta_a) - Math.logbeta(1 + j, beta_b) -
+              Math.logbeta(alpha_c, beta_c))
+          end
+        end
+
+        1 - prob_b_beats_a(alpha_c, beta_c, alpha_a, beta_a) -
+          prob_b_beats_a(alpha_c, beta_c, alpha_b, beta_b) + total
       end
   end
 end

data/lib/field_test/helpers.rb

@@ -24,6 +24,18 @@ module FieldTest
       exp.convert(participants)
     end
 
+    def field_test_experiments(options = {})
+      participants = field_test_participants(options)
+      memberships = FieldTest::Membership.where(participant: participants).group_by(&:participant)
+      experiments = {}
+      participants.each do |participant|
+        memberships[participant].each do |membership|
+          experiments[membership.experiment] ||= membership.variant
+        end
+      end
+      experiments
+    end
+
     def field_test_participants(options = {})
       participants = []
 
@@ -48,7 +60,7 @@ module FieldTest
         end
       end
 
-      participants
+      FieldTest::Participant.standardize(participants)
     end
   end
 end

data/lib/field_test/participant.rb

@@ -0,0 +1,7 @@
+module FieldTest
+  class Participant
+    def self.standardize(participants)
+      Array(participants).map { |v| v.respond_to?(:model_name) ? "#{v.model_name.name}:#{v.id}" : v.to_s }
+    end
+  end
+end

data/lib/field_test/version.rb

@@ -1,3 +1,3 @@
 module FieldTest
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/generators/field_test/templates/config.yml

@@ -1,7 +1,7 @@
 experiments:
   button_color:
     variants:
-      - control
       - red
       - green
-    # winner: red
+      - blue
+    # winner: green

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: field_test
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-14 00:00:00.000000000 Z
+date: 2016-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: distribution
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -79,12 +93,18 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- app/controllers/field_test/home_controller.rb
 - app/models/field_test/membership.rb
+- app/views/field_test/home/_experiments.html.erb
+- app/views/field_test/home/index.html.erb
+- app/views/layouts/field_test/application.html.erb
+- config/routes.rb
 - field_test.gemspec
 - lib/field_test.rb
 - lib/field_test/engine.rb
 - lib/field_test/experiment.rb
 - lib/field_test/helpers.rb
+- lib/field_test/participant.rb
 - lib/field_test/version.rb
 - lib/generators/field_test/install_generator.rb
 - lib/generators/field_test/templates/config.yml