field_test 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8bbd74bb28099c186bacc0ea7b195c7d89a7e35
-  data.tar.gz: e87355a520385d34192ab200b80411a4bbc70310
+  metadata.gz: 715ea396faed10c5b142f003eb76405d8ba34f01
+  data.tar.gz: 43a830f9996b974a917734f39641d8783b9c33c9
 SHA512:
-  metadata.gz: 6f717922757f35cd62c835beb58bde3bce74fb326ca1965ddb5bbdd03811d1433fd99f1e677738b52d12f3c9158919faf832f99427d507ce5f23423b03757ccc
-  data.tar.gz: 5912035f52bfb111970e1fcf3f6bf73a9c4dbe55334e04b39a1ba9e456481ffe23f5838974989f45f560a16960b6f2a835bce2a0603c34991167fe39caa75230
+  metadata.gz: fee3d99177f6e2044bc38bba2a557e4832b9a402b4743400a8c6a47cff272885417100caf7522ef48ef62f2641a71790e1107cc4885d079d5624673a26178e18
+  data.tar.gz: 07a3450cd3e1ad4da4a909841ea7532c7683b65a366907345f186bd261bac53721d1a4b603f97c0f8e541773dc39d2e7398a41655c0811effeece190b29a9741

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.0
+
+- Better web UI
+- Removed `cookie:` prefix for unknown participants
+
 ## 0.1.2
 
 - Exclude bots

data/README.md

@@ -3,8 +3,8 @@
 :maple_leaf: A/B testing for Rails
 
 - Designed for web and email
+- Comes with a [nice dashboard](https://fieldtest.dokkuapp.com/)
 - Seamlessly handles the transition from anonymous visitor to logged in user
-- Results are stored in your database
 
 Uses [Bayesian methods](http://www.evanmiller.org/bayesian-ab-testing.html) to evaluate results so you don’t need to choose a sample size ahead of time.
 
@@ -30,7 +30,7 @@ mount FieldTest::Engine, at: "field_test"
 
 Be sure to [secure the dashboard](#security) in production.
 
-![Screenshot](https://ankane.github.io/field_test/screenshot5.png)
+![Screenshot](https://ankane.github.io/field_test/screenshot6.png)
 
 ## Getting Started
 
@@ -57,13 +57,6 @@ When someone converts, record it with:
 field_test_converted(:button_color)
 ```
 
-Get the results with:
-
-```ruby
-experiment = FieldTest::Experiment.find(:button_color)
-experiment.results
-```
-
 When an experiment is over, specify a winner:
 
 ```yml
@@ -86,19 +79,7 @@ Assign a specific variant to a user with:
 
 ```ruby
 experiment = FieldTest::Experiment.find(:button_color)
-experiment.variant(user, variant: "red")
-```
-
-Specify a participant with:
-
-```ruby
-field_test(:button_color, participant: "test@example.org")
-```
-
-You can pass an object as well.
-
-```ruby
-field_test(:button_color, participant: user)
+experiment.variant(participant, variant: "red")
 ```
 
 ## Config
@@ -110,13 +91,24 @@ exclude:
   bots: false
 ```
 
-Keep track of when experiments started and ended.
+Keep track of when experiments started and ended. Use any format `Time.parse` accepts.
+
+```yml
+experiments:
+  button_color:
+    started_at: Dec 1, 2016 8 am PST
+    ended_at: Dec 8, 2016 2 pm PST
+```
+
+Add a friendlier name and description with:
 
 ```yml
 experiments:
   button_color:
-    started_at: 2016-12-01 14:00:00
-    ended_at: 2016-12-08 14:00:00
+    name: Buttons!
+    description: >
+      Different button colors
+      for the landing page.
 ```
 
 By default, variants are given the same probability of being selected. Change this with:
@@ -132,6 +124,14 @@ experiments:
       - 10
 ```
 
+If the dashboard gets slow, you can make it faster with:
+
+```yml
+cache: true
+```
+
+This will use the Rails cache to speed up winning probability calculations.
+
 ## Funnels
 
 For advanced funnels, we recommend an analytics platform like [Ahoy](https://github.com/ankane/ahoy) or [Mixpanel](https://mixpanel.com/).
@@ -146,6 +146,14 @@ to get all experiments and variants for a participant, and pass them as properti
 
 ## Security
 
+#### Devise
+
+```ruby
+authenticate :user, -> (user) { user.admin? } do
+  mount FieldTest::Engine, at: "field_test"
+end
+```
+
 #### Basic Authentication
 
 Set the following variables in your environment or an initializer.
@@ -155,14 +163,6 @@ ENV["FIELD_TEST_USERNAME"] = "moonrise"
 ENV["FIELD_TEST_PASSWORD"] = "kingdom"
 ```
 
-#### Devise
-
-```ruby
-authenticate :user, -> (user) { user.admin? } do
-  mount FieldTest::Engine, at: "field_test"
-end
-```
-
 ## Credits
 
 A huge thanks to [Evan Miller](http://www.evanmiller.org/) for deriving the Bayesian formulas.

data/app/controllers/field_test/{home_controller.rb → base_controller.rb}

@@ -1,13 +1,9 @@
 module FieldTest
-  class HomeController < ActionController::Base
+  class BaseController < ActionController::Base
     layout "field_test/application"
 
     protect_from_forgery
 
     http_basic_authenticate_with name: ENV["FIELD_TEST_USERNAME"], password: ENV["FIELD_TEST_PASSWORD"] if ENV["FIELD_TEST_PASSWORD"]
-
-    def index
-      @active_experiments, @completed_experiments = FieldTest::Experiment.all.sort_by(&:id).partition { |e| e.active? }
-    end
   end
 end

data/app/controllers/field_test/experiments_controller.rb

@@ -0,0 +1,18 @@
+module FieldTest
+  class ExperimentsController < BaseController
+    def index
+      @active_experiments, @completed_experiments = FieldTest::Experiment.all.sort_by(&:id).partition { |e| e.active? }
+    end
+
+    def show
+      @experiment = FieldTest::Experiment.find(params[:id])
+
+      @per_page = 200
+      @page = [1, params[:page].to_i].max
+      offset = (@page - 1) * @per_page
+      @memberships = @experiment.memberships.order(created_at: :desc).limit(@per_page).offset(offset).to_a
+    rescue FieldTest::ExperimentNotFound
+      raise ActionController::RoutingError, "Experiment not found"
+    end
+  end
+end

data/app/controllers/field_test/memberships_controller.rb

@@ -0,0 +1,15 @@
+module FieldTest
+  class MembershipsController < BaseController
+    def update
+      membership = FieldTest::Membership.find(params[:id])
+      membership.update_attributes(membership_params)
+      redirect_to participant_path(membership.participant)
+    end
+
+    private
+
+      def membership_params
+        params.require(:membership).permit(:variant)
+      end
+  end
+end

data/app/controllers/field_test/participants_controller.rb

@@ -0,0 +1,7 @@
+module FieldTest
+  class ParticipantsController < BaseController
+    def show
+      @participant = params[:id]
+    end
+  end
+end

data/app/views/field_test/{home → experiments}/_experiments.html.erb

@@ -1,7 +1,16 @@
 <% experiments.each do |experiment| %>
   <% results = experiment.results %>
 
-  <h2><%= experiment.name %></h2>
+  <h2>
+    <%= experiment.name %>
+    <small><%= link_to "Details", experiment_path(experiment.id) %></small>
+  </h2>
+
+
+  <% if experiment.description %>
+    <p class="description"><%= experiment.description %></p>
+  <% end %>
+
   <table>
     <thead>
       <tr>
@@ -18,7 +27,7 @@
           <td>
             <%= variant %>
             <% if variant == experiment.winner %>
-              <span style="color: #999;">winner</span>
+              <span class="check">✓</span>
             <% end %>
           </td>
           <td><%= result[:participated] %></td>
@@ -32,7 +41,11 @@
           </td>
           <td>
             <% if result[:prob_winning] %>
-              <%= (100.0 * result[:prob_winning]).round %>%
+              <% if result[:prob_winning] < 0.01 %>
+                &lt; 1%
+              <% else %>
+                <%= (100.0 * result[:prob_winning]).round %>%
+              <% end %>
             <% end %>
           </td>
         </tr>

data/app/views/field_test/experiments/show.html.erb

@@ -0,0 +1,52 @@
+<p>
+  <%= link_to root_path do %>
+    &laquo; Experiments
+  <% end %>
+</p>
+
+<h1><%= @experiment.name %></h1>
+
+<table>
+  <thead>
+    <tr>
+      <th>Participant</th>
+      <th style="width: 20%;">Variant</th>
+      <th style="width: 20%;">Converted</th>
+      <th style="width: 20%;">Started</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @memberships.each do |membership| %>
+      <tr>
+        <td><%= link_to membership.participant, participant_path(membership.participant) %></td>
+        <td><%= membership.variant %></td>
+        <td>
+          <% if membership.converted %>
+            <span class="check">✓</span>
+          <% end %>
+        </td>
+        <td>
+          <% if membership.created_at > 1.day.ago %>
+            <%= time_ago_in_words(membership.created_at, include_seconds: true) %> ago
+          <% else %>
+            <%= membership.created_at.to_formatted_s(:short) %>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<p class="pagination">
+  <% unless @page == 1 %>
+    <%= link_to experiment_path(@experiment.id, page: @page - 1) do %>
+      &laquo; Prev
+    <% end %>
+  <% end %>
+  <!-- there may not be a next page, but don't want another DB query -->
+  <% if @memberships.size == @per_page %>
+    <%= link_to experiment_path(@experiment.id, page: @page + 1) do %>
+      Next &raquo;
+    <% end %>
+  <% end %>
+</p>

data/app/views/field_test/participants/show.html.erb

@@ -0,0 +1,47 @@
+<p>
+  <%= link_to root_path do %>
+    &laquo; Experiments
+  <% end %>
+</p>
+
+<h1><%= @participant %></h1>
+
+<table>
+  <thead>
+    <tr>
+      <th>Experiment</th>
+      <th style="width: 20%;">Variant</th>
+      <th style="width: 20%;">Converted</th>
+      <th style="width: 20%;">Started</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% FieldTest::Membership.where(participant: @participant).each do |membership| %>
+      <tr>
+        <td><%= link_to membership.experiment, experiment_path(membership.experiment) %></td>
+        <td>
+          <% experiment = FieldTest::Experiment.find(membership.experiment) rescue nil %>
+          <% if experiment %>
+            <%= form_for membership do |f| %>
+              <%= f.select "variant", options_for_select(FieldTest::Experiment.find(membership.experiment).variants.map { |v| [v, v] }, membership.variant), {}, onchange: "this.form.submit()" %>
+            <% end %>
+          <% else %>
+            <%= membership.variant %>
+          <% end %>
+        </td>
+        <td>
+          <% if membership.converted %>
+            <span class="check">✓</span>
+          <% end %>
+        </td>
+        <td>
+          <% if membership.created_at > 1.day.ago %>
+            <%= time_ago_in_words(membership.created_at, include_seconds: true) %> ago
+          <% else %>
+            <%= membership.created_at.to_formatted_s(:short) %>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/app/views/layouts/field_test/application.html.erb

@@ -7,15 +7,21 @@
         margin: 0;
         padding: 20px;
         font-family: "Helvetica Neue", Arial, Helvetica, sans-serif;
-        font-size: 14px;
+        font-size: 16px;
         line-height: 1.4;
         color: #333;
       }
 
+      a, a:visited, a:active {
+        color: #08c;
+        text-decoration: none;
+      }
+
       table {
         width: 100%;
         border-collapse: collapse;
         border-spacing: 0;
+        font-size: 16px;
         margin-bottom: 20px;
       }
 
@@ -31,6 +37,28 @@
       td {
         border-top: solid 1px #ddd;
       }
+
+      h2 small {
+        font-size: 16px;
+        font-weight: normal;
+      }
+
+      .description {
+        color: #999;
+      }
+
+      .check {
+        color: #5cb85c;
+      }
+
+      .pagination {
+        text-align: center;
+      }
+
+      .pagination a {
+        padding-left: 10px;
+        padding-right: 10px;
+      }
     </style>
   </head>
   <body>

data/config/routes.rb

@@ -1,3 +1,6 @@
 FieldTest::Engine.routes.draw do
-  root "home#index"
+  resources :experiments, only: [:show]
+  resources :memberships, only: [:update]
+  get "participants/:id", to: "participants#show", constraints: {id: /.+/}, as: :participant
+  root "experiments#index"
 end

data/field_test.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
 end

data/lib/field_test.rb

@@ -1,7 +1,9 @@
 require "distribution/math_extension"
 require "browser"
+require "active_support"
+require "field_test/calculations"
 require "field_test/experiment"
-require "field_test/engine"
+require "field_test/engine" if defined?(Rails)
 require "field_test/helpers"
 require "field_test/participant"
 require "field_test/version"
@@ -22,6 +24,10 @@ module FieldTest
     config = self.config # dev performance
     config["exclude"] && config["exclude"]["bots"]
   end
+
+  def self.cache
+    config["cache"]
+  end
 end
 
 ActiveSupport.on_load(:action_controller) do

data/lib/field_test/calculations.rb

@@ -0,0 +1,59 @@
+# formulas from
+# http://www.evanmiller.org/bayesian-ab-testing.html
+
+module FieldTest
+  module Calculations
+    def self.prob_b_beats_a(alpha_a, beta_a, alpha_b, beta_b)
+      total = 0.0
+
+      # for performance
+      logbeta_aa_ba = Math.logbeta(alpha_a, beta_a)
+      beta_ba = beta_b + beta_a
+
+      0.upto(alpha_b - 1) do |i|
+        total += Math.exp(Math.logbeta(alpha_a + i, beta_ba) -
+          Math.log(beta_b + i) - Math.logbeta(1 + i, beta_b) -
+          logbeta_aa_ba)
+      end
+
+      total
+    end
+
+    def self.prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
+      total = 0.0
+
+      # for performance
+      logbeta_ac_bc = Math.logbeta(alpha_c, beta_c)
+      abc = beta_a + beta_b + beta_c
+      log_bb_j = []
+      logbeta_j_bb = []
+      0.upto(alpha_b - 1) do |j|
+        log_bb_j[j] = Math.log(beta_b + j)
+        logbeta_j_bb[j] = Math.logbeta(1 + j, beta_b)
+      end
+
+      logbeta_ac_i_j = []
+      0.upto(alpha_a - 1) do |i|
+        0.upto(alpha_b - 1) do |j|
+          logbeta_ac_i_j[i + j] ||= Math.logbeta(alpha_c + i + j, abc)
+        end
+      end
+
+      0.upto(alpha_a - 1) do |i|
+        # for performance
+        log_ba_i = Math.log(beta_a + i)
+        logbeta_i_ba = Math.logbeta(1 + i, beta_a)
+
+        0.upto(alpha_b - 1) do |j|
+          total += Math.exp(logbeta_ac_i_j[i + j] -
+            log_ba_i - log_bb_j[j] -
+            logbeta_i_ba - logbeta_j_bb[j] -
+            logbeta_ac_bc)
+        end
+      end
+
+      1 - prob_b_beats_a(alpha_c, beta_c, alpha_a, beta_a) -
+        prob_b_beats_a(alpha_c, beta_c, alpha_b, beta_b) + total
+    end
+  end
+end

data/lib/field_test/experiment.rb

@@ -1,11 +1,12 @@
 module FieldTest
   class Experiment
-    attr_reader :id, :name, :variants, :weights, :winner, :started_at, :ended_at
+    attr_reader :id, :name, :description, :variants, :weights, :winner, :started_at, :ended_at
 
     def initialize(attributes)
       attributes = attributes.symbolize_keys
       @id = attributes[:id]
       @name = attributes[:name] || @id.to_s.titleize
+      @description = attributes[:description]
       @variants = attributes[:variants]
       @weights = @variants.size.times.map { |i| attributes[:weights].to_a[i] || 1 }
       @winner = attributes[:winner]
@@ -33,6 +34,17 @@ module FieldTest
       if membership.changed?
         begin
           membership.save!
+
+          # log it!
+          info = {
+            experiment: id,
+            variant: membership.variant,
+            participant: membership.participant
+          }.merge(options.slice(:ip, :user_agent))
+
+          # sorta logfmt :)
+          info = info.map { |k, v| v = "\"#{v}\"" if k == :user_agent; "#{k}=#{v}" }.join(" ")
+          Rails.logger.info "[field test] #{info}"
         rescue ActiveRecord::RecordNotUnique
           membership = memberships.find_by(participant: participants.first)
         end
@@ -75,10 +87,10 @@ module FieldTest
         }
       end
       case variants.size
-      when 1
-        results[variants[0]][:prob_winning] = 1
-      when 2, 3
-        variants.size.times do |i|
+      when 1, 2, 3
+        total = 0.0
+
+        (variants.size - 1).times do |i|
           c = results.values[i]
           b = results.values[(i + 1) % variants.size]
           a = results.values[(i + 2) % variants.size]
@@ -90,13 +102,23 @@ module FieldTest
           alpha_c = 1 + c[:converted]
           beta_c = 1 + c[:participated] - c[:converted]
 
-          results[variants[i]][:prob_winning] =
+          # TODO calculate this incrementally by caching intermediate results
+          prob_winning =
             if variants.size == 2
-              prob_b_beats_a(alpha_b, beta_b, alpha_c, beta_c)
+              cache_fetch ["field_test", "prob_b_beats_a", alpha_b, beta_b, alpha_c, beta_c] do
+                Calculations.prob_b_beats_a(alpha_b, beta_b, alpha_c, beta_c)
+              end
             else
-              prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
+              cache_fetch ["field_test", "prob_c_beats_a_and_b", alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c] do
+                Calculations.prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
+              end
             end
+
+          results[variants[i]][:prob_winning] = prob_winning
+          total += prob_winning
         end
+
+        results[variants.last][:prob_winning] = 1 - total
       end
       results
     end
@@ -140,33 +162,12 @@ module FieldTest
         variants.last
       end
 
-      # formula from
-      # http://www.evanmiller.org/bayesian-ab-testing.html
-      def prob_b_beats_a(alpha_a, beta_a, alpha_b, beta_b)
-        total = 0.0
-
-        0.upto(alpha_b - 1) do |i|
-          total += Math.exp(Math.logbeta(alpha_a + i, beta_b + beta_a) -
-            Math.log(beta_b + i) - Math.logbeta(1 + i, beta_b) -
-            Math.logbeta(alpha_a, beta_a))
-        end
-
-        total
-      end
-
-      def prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
-        total = 0.0
-        0.upto(alpha_a - 1) do |i|
-          0.upto(alpha_b - 1) do |j|
-            total += Math.exp(Math.logbeta(alpha_c + i + j, beta_a + beta_b + beta_c) -
-              Math.log(beta_a + i) - Math.log(beta_b + j) -
-              Math.logbeta(1 + i, beta_a) - Math.logbeta(1 + j, beta_b) -
-              Math.logbeta(alpha_c, beta_c))
-          end
+      def cache_fetch(key)
+        if FieldTest.cache
+          Rails.cache.fetch(key.join("/")) { yield }
+        else
+          yield
         end
-
-        1 - prob_b_beats_a(alpha_c, beta_c, alpha_a, beta_a) -
-          prob_b_beats_a(alpha_c, beta_c, alpha_b, beta_b) + total
       end
   end
 end

data/lib/field_test/helpers.rb

@@ -13,6 +13,9 @@ module FieldTest
         if FieldTest.exclude_bots?
           options[:exclude] = Browser.new(request.user_agent).bot?
         end
+
+        options[:ip] = request.remote_ip
+        options[:user_agent] = request.user_agent
       end
 
       # cache results for request
@@ -54,13 +57,19 @@ module FieldTest
         if try(:request)
           # use cookie
           cookie_key = "field_test"
+
           token = cookies[cookie_key]
+          token = token.gsub(/[^a-z0-9\-]/i, "") if token
+
           if participants.empty? && !token
             token = SecureRandom.uuid
             cookies[cookie_key] = {value: token, expires: 30.days.from_now}
           end
           if token
-            participants << "cookie:#{token.gsub(/[^a-z0-9\-]/i, "")}"
+            participants << token
+
+            # backwards compatibility
+            participants << "cookie:#{token}"
           end
         end
 

data/lib/field_test/version.rb

@@ -1,3 +1,3 @@
 module FieldTest
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/generators/field_test/templates/memberships.rb

@@ -10,5 +10,6 @@ class <%= migration_class_name %> < ActiveRecord::Migration
 
     add_index :field_test_memberships, [:experiment, :participant], unique: true
     add_index :field_test_memberships, :participant
+    add_index :field_test_memberships, [:experiment, :created_at]
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: field_test
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-17 00:00:00.000000000 Z
+date: 2016-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - andrew@chartkick.com
@@ -107,14 +121,20 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- app/controllers/field_test/home_controller.rb
+- app/controllers/field_test/base_controller.rb
+- app/controllers/field_test/experiments_controller.rb
+- app/controllers/field_test/memberships_controller.rb
+- app/controllers/field_test/participants_controller.rb
 - app/models/field_test/membership.rb
-- app/views/field_test/home/_experiments.html.erb
-- app/views/field_test/home/index.html.erb
+- app/views/field_test/experiments/_experiments.html.erb
+- app/views/field_test/experiments/index.html.erb
+- app/views/field_test/experiments/show.html.erb
+- app/views/field_test/participants/show.html.erb
 - app/views/layouts/field_test/application.html.erb
 - config/routes.rb
 - field_test.gemspec
 - lib/field_test.rb
+- lib/field_test/calculations.rb
 - lib/field_test/engine.rb
 - lib/field_test/experiment.rb
 - lib/field_test/helpers.rb