field_test 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 715ea396faed10c5b142f003eb76405d8ba34f01
-  data.tar.gz: 43a830f9996b974a917734f39641d8783b9c33c9
+  metadata.gz: 1f722991205f31c2faf4c59852d717ac229f3043
+  data.tar.gz: 18b45c92adbcec168c41e051ea1c6613c6dadbfc
 SHA512:
-  metadata.gz: fee3d99177f6e2044bc38bba2a557e4832b9a402b4743400a8c6a47cff272885417100caf7522ef48ef62f2641a71790e1107cc4885d079d5624673a26178e18
-  data.tar.gz: 07a3450cd3e1ad4da4a909841ea7532c7683b65a366907345f186bd261bac53721d1a4b603f97c0f8e541773dc39d2e7398a41655c0811effeece190b29a9741
+  metadata.gz: d18d2688655ebd5a1d523b3ec8452fd96701e1a63b6cb7f93eae395fd2bfae39d7bf1c716e6617cf9a96e47e7fd129035ef31f3243e928653c52839dba499a9b
+  data.tar.gz: 8cedb82ac15520e091e46cc82ec7fa21faa41ec7ed0fc60f0d351b82948cb6c823ecaca645423893feaa5a1a4dc828e173a1255fd39bffe2401d84a868fc7fbf

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1
+
+- Added support for multiple goals
+
 ## 0.2.0
 
 - Better web UI

data/README.md

@@ -3,10 +3,10 @@
 :maple_leaf: A/B testing for Rails
 
 - Designed for web and email
-- Comes with a [nice dashboard](https://fieldtest.dokkuapp.com/)
+- Comes with a [handy dashboard](https://fieldtest.dokkuapp.com/)
 - Seamlessly handles the transition from anonymous visitor to logged in user
 
-Uses [Bayesian methods](http://www.evanmiller.org/bayesian-ab-testing.html) to evaluate results so you don’t need to choose a sample size ahead of time.
+Uses [Bayesian statistics](http://www.evanmiller.org/bayesian-ab-testing.html) to evaluate results so you don’t need to choose a sample size ahead of time.
 
 ## Installation
 
@@ -62,7 +62,7 @@ When an experiment is over, specify a winner:
 ```yml
 experiments:
   button_color:
-    winner: red
+    winner: green
 ```
 
 All calls to `field_test` will now return the winner, and metrics will stop being recorded.
@@ -72,16 +72,18 @@ All calls to `field_test` will now return the winner, and metrics will stop bein
 You can specify a variant with query parameters to make testing easier
 
 ```
-http://localhost:3000/?field_test[button_color]=red
+?field_test[button_color]=green
 ```
 
 Assign a specific variant to a user with:
 
 ```ruby
 experiment = FieldTest::Experiment.find(:button_color)
-experiment.variant(participant, variant: "red")
+experiment.variant(participant, variant: "green")
 ```
 
+You can also change a user’s variant from the dashboard.
+
 ## Config
 
 By default, bots are returned the first variant and excluded from metrics. Change this with:
@@ -91,7 +93,7 @@ exclude:
   bots: false
 ```
 
-Keep track of when experiments started and ended. Use any format `Time.parse` accepts.
+Keep track of when experiments started and ended. Use any format `Time.parse` accepts. Variants assigned outside this window are not included in metrics.
 
 ```yml
 experiments:
@@ -120,8 +122,8 @@ experiments:
       - red
       - blue
     weights:
-      - 90
-      - 10
+      - 85
+      - 15
 ```
 
 If the dashboard gets slow, you can make it faster with:
@@ -134,9 +136,33 @@ This will use the Rails cache to speed up winning probability calculations.
 
 ## Funnels
 
-For advanced funnels, we recommend an analytics platform like [Ahoy](https://github.com/ankane/ahoy) or [Mixpanel](https://mixpanel.com/).
+You can set multiple goals for an experiment to track conversions at different parts of the funnel. First, run:
+
+```sh
+rails g field_test:events
+```
+
+And add to your config:
+
+```yml
+experiments:
+  button_color:
+    goals:
+      - signed_up
+      - ordered
+```
+
+Specify a goal during conversion with:
+
+```ruby
+field_test_converted(:button_color, goal: "ordered")
+```
+
+The results for all goals will appear on the dashboard.
+
+### Advanced
 
-You can use:
+For advanced funnels, we recommend an analytics platform like [Ahoy](https://github.com/ankane/ahoy) or [Mixpanel](https://mixpanel.com/). You can use:
 
 ```ruby
 field_test_experiments

data/app/controllers/field_test/experiments_controller.rb

@@ -11,6 +11,13 @@ module FieldTest
       @page = [1, params[:page].to_i].max
       offset = (@page - 1) * @per_page
       @memberships = @experiment.memberships.order(created_at: :desc).limit(@per_page).offset(offset).to_a
+
+      @events =
+        if FieldTest.events_supported?
+          @experiment.events.where(field_test_membership_id: @memberships.map(&:id)).group(:field_test_membership_id, :name).count
+        else
+          {}
+        end
     rescue FieldTest::ExperimentNotFound
       raise ActionController::RoutingError, "Experiment not found"
     end

data/app/controllers/field_test/participants_controller.rb

@@ -2,6 +2,15 @@ module FieldTest
   class ParticipantsController < BaseController
     def show
       @participant = params[:id]
+      # TODO better ordering
+      @memberships = FieldTest::Membership.where(participant: @participant).order(:id)
+
+      @events =
+        if FieldTest.events_supported?
+          FieldTest::Event.where(field_test_membership_id: @memberships.map(&:id)).group(:field_test_membership_id, :name).count
+        else
+          {}
+        end
     end
   end
 end

data/app/models/field_test/event.rb

@@ -0,0 +1,8 @@
+module FieldTest
+  class Event < ActiveRecord::Base
+    self.table_name = "field_test_events"
+
+    belongs_to :field_test_membership, class_name: "FieldTest::Membership"
+    validates :name, presence: true
+  end
+end

data/app/models/field_test/membership.rb

@@ -2,6 +2,8 @@ module FieldTest
   class Membership < ActiveRecord::Base
     self.table_name = "field_test_memberships"
 
+    has_many :events, class_name: "FieldTest::Event"
+
     validates :participant, presence: true
     validates :experiment, presence: true
     validates :variant, presence: true

data/app/views/field_test/experiments/_experiments.html.erb

@@ -1,55 +1,60 @@
 <% experiments.each do |experiment| %>
-  <% results = experiment.results %>
-
   <h2>
     <%= experiment.name %>
     <small><%= link_to "Details", experiment_path(experiment.id) %></small>
   </h2>
 
-
   <% if experiment.description %>
     <p class="description"><%= experiment.description %></p>
   <% end %>
 
-  <table>
-    <thead>
-      <tr>
-        <th>Variant</th>
-        <th style="width: 20%;">Participants</th>
-        <th style="width: 20%;">Conversions</th>
-        <th style="width: 20%;">Conversion Rate</th>
-        <th style="width: 20%;">Prob Winning</th>
-      </tr>
-    </thead>
-    <tbody>
-      <% results.each do |variant, result| %>
+  <% experiment.goals.each do |goal| %>
+    <% results = experiment.results(goal: goal) %>
+
+    <% if experiment.multiple_goals? %>
+      <h3><%= goal.titleize %></h3>
+    <% end %>
+
+    <table>
+      <thead>
         <tr>
-          <td>
-            <%= variant %>
-            <% if variant == experiment.winner %>
-              <span class="check">✓</span>
-            <% end %>
-          </td>
-          <td><%= result[:participated] %></td>
-          <td><%= result[:converted] %></td>
-          <td>
-            <% if result[:conversion_rate] %>
-              <%= (100.0 * result[:conversion_rate]).round %>%
-            <% else %>
-              -
-            <% end %>
-          </td>
-          <td>
-            <% if result[:prob_winning] %>
-              <% if result[:prob_winning] < 0.01 %>
-                &lt; 1%
+          <th>Variant</th>
+          <th style="width: 20%;">Participants</th>
+          <th style="width: 20%;">Conversions</th>
+          <th style="width: 20%;">Conversion Rate</th>
+          <th style="width: 20%;">Prob Winning</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% results.each do |variant, result| %>
+          <tr>
+            <td>
+              <%= variant %>
+              <% if variant == experiment.winner %>
+                <span class="check">✓</span>
+              <% end %>
+            </td>
+            <td><%= result[:participated] %></td>
+            <td><%= result[:converted] %></td>
+            <td>
+              <% if result[:conversion_rate] %>
+                <%= (100.0 * result[:conversion_rate]).round %>%
               <% else %>
-                <%= (100.0 * result[:prob_winning]).round %>%
+                -
               <% end %>
-            <% end %>
-          </td>
-        </tr>
-      <% end %>
-    </tbody>
-  </table>
+            </td>
+            <td>
+              <% if result[:prob_winning] %>
+                <% if result[:prob_winning] < 0.01 %>
+                  &lt; 1%
+                <% else %>
+                  <%= (100.0 * result[:prob_winning]).round %>%
+                <% end %>
+              <% end %>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
 <% end %>

data/app/views/field_test/experiments/show.html.erb

@@ -6,6 +6,10 @@
 
 <h1><%= @experiment.name %></h1>
 
+<% if @experiment.description %>
+  <p class="description"><%= @experiment.description %></p>
+<% end %>
+
 <table>
   <thead>
     <tr>
@@ -21,7 +25,20 @@
         <td><%= link_to membership.participant, participant_path(membership.participant) %></td>
         <td><%= membership.variant %></td>
         <td>
-          <% if membership.converted %>
+          <% converted = false %>
+          <% @experiment.goals.each do |goal| %>
+            <% if @events[[membership.id, goal]] %>
+              <% converted = true %>
+              <div>
+                <span class="check">✓</span>
+                <% if @experiment.multiple_goals? %>
+                  <%= goal.titleize %>
+                <% end %>
+              </div>
+            <% end %>
+          <% end %>
+
+          <% if !converted && membership.try(:converted) %>
             <span class="check">✓</span>
           <% end %>
         </td>

data/app/views/field_test/participants/show.html.erb

@@ -16,7 +16,7 @@
     </tr>
   </thead>
   <tbody>
-    <% FieldTest::Membership.where(participant: @participant).each do |membership| %>
+    <% @memberships.each do |membership| %>
       <tr>
         <td><%= link_to membership.experiment, experiment_path(membership.experiment) %></td>
         <td>
@@ -30,7 +30,22 @@
           <% end %>
         </td>
         <td>
-          <% if membership.converted %>
+          <% converted = false %>
+          <% if experiment %>
+          <% experiment.goals.each do |goal| %>
+              <% if @events[[membership.id, goal]] %>
+                <% converted = true %>
+                <div>
+                  <span class="check">✓</span>
+                  <% if experiment.multiple_goals? %>
+                    <%= goal.titleize %>
+                  <% end %>
+                </div>
+              <% end %>
+            <% end %>
+          <% end %>
+
+          <% if !converted && membership.try(:converted) %>
             <span class="check">✓</span>
           <% end %>
         </td>

data/app/views/layouts/field_test/application.html.erb

@@ -36,6 +36,7 @@
 
       td {
         border-top: solid 1px #ddd;
+        vertical-align: top;
       }
 
       h2 small {
@@ -43,6 +44,20 @@
         font-weight: normal;
       }
 
+      form {
+        margin: 0;
+      }
+
+      ul {
+        margin: 0;
+        padding: 0;
+        list-style-type: none;
+      }
+
+      li {
+        margin-bottom: 0;
+      }
+
       .description {
         color: #999;
       }

data/lib/field_test.rb

@@ -1,4 +1,3 @@
-require "distribution/math_extension"
 require "browser"
 require "active_support"
 require "field_test/calculations"
@@ -28,6 +27,20 @@ module FieldTest
   def self.cache
     config["cache"]
   end
+
+  def self.events_supported?
+    unless defined?(@events_supported)
+      connection = FieldTest::Membership.connection
+      table_name = "field_test_events"
+      @events_supported =
+        if connection.respond_to?(:data_source_exists?)
+          connection.data_source_exists?(table_name)
+        else
+          connection.table_exists?(table_name)
+        end
+    end
+    @events_supported
+  end
 end
 
 ActiveSupport.on_load(:action_controller) do

data/lib/field_test/calculations.rb

@@ -1,6 +1,7 @@
+require "distribution/math_extension"
+
 # formulas from
 # http://www.evanmiller.org/bayesian-ab-testing.html
-
 module FieldTest
   module Calculations
     def self.prob_b_beats_a(alpha_a, beta_a, alpha_b, beta_b)
@@ -27,14 +28,12 @@ module FieldTest
       abc = beta_a + beta_b + beta_c
       log_bb_j = []
       logbeta_j_bb = []
+      logbeta_ac_i_j = []
       0.upto(alpha_b - 1) do |j|
         log_bb_j[j] = Math.log(beta_b + j)
         logbeta_j_bb[j] = Math.logbeta(1 + j, beta_b)
-      end
 
-      logbeta_ac_i_j = []
-      0.upto(alpha_a - 1) do |i|
-        0.upto(alpha_b - 1) do |j|
+        0.upto(alpha_a - 1) do |i|
           logbeta_ac_i_j[i + j] ||= Math.logbeta(alpha_c + i + j, abc)
         end
       end

data/lib/field_test/experiment.rb

@@ -1,6 +1,6 @@
 module FieldTest
   class Experiment
-    attr_reader :id, :name, :description, :variants, :weights, :winner, :started_at, :ended_at
+    attr_reader :id, :name, :description, :variants, :weights, :winner, :started_at, :ended_at, :goals
 
     def initialize(attributes)
       attributes = attributes.symbolize_keys
@@ -12,6 +12,8 @@ module FieldTest
       @winner = attributes[:winner]
       @started_at = Time.zone.parse(attributes[:started_at].to_s) if attributes[:started_at]
       @ended_at = Time.zone.parse(attributes[:ended_at].to_s) if attributes[:ended_at]
+      @goals = attributes[:goals] || ["conversion"]
+      @use_events = attributes[:use_events]
     end
 
     def variant(participants, options = {})
@@ -53,14 +55,26 @@ module FieldTest
       membership.try(:variant) || variants.first
     end
 
-    def convert(participants)
+    def convert(participants, goal: nil)
+      goal ||= goals.first
+
       participants = FieldTest::Participant.standardize(participants)
       check_participants(participants)
       membership = membership_for(participants)
 
       if membership
-        membership.converted = true
-        membership.save! if membership.changed?
+        if membership.respond_to?(:converted)
+          membership.converted = true
+          membership.save! if membership.changed?
+        end
+
+        if use_events?
+          FieldTest::Event.create!(
+            name: goal,
+            field_test_membership_id: membership.id
+          )
+        end
+
         true
       else
         false
@@ -71,11 +85,36 @@ module FieldTest
       FieldTest::Membership.where(experiment: id)
     end
 
-    def results
-      data = memberships.group(:variant).group(:converted)
-      data = data.where("created_at >= ?", started_at) if started_at
-      data = data.where("created_at <= ?", ended_at) if ended_at
-      data = data.count
+    def events
+      FieldTest::Event.joins(:field_test_membership).where(field_test_memberships: {experiment: id})
+    end
+
+    def multiple_goals?
+      goals.size > 1
+    end
+
+    def results(goal: nil)
+      goal ||= goals.first
+
+      relation = memberships.group(:variant)
+      relation = relation.where("created_at >= ?", started_at) if started_at
+      relation = relation.where("created_at <= ?", ended_at) if ended_at
+
+      if use_events?
+        data = {}
+        sql =
+          relation.joins("LEFT JOIN field_test_events ON field_test_events.field_test_membership_id = field_test_memberships.id")
+            .select("variant, COUNT(DISTINCT participant) AS participated, COUNT(DISTINCT field_test_membership_id) AS converted")
+            .where(field_test_events: {name: goal})
+
+        FieldTest::Membership.connection.select_all(sql).each do |row|
+          data[[row["variant"], true]] = row["converted"].to_i
+          data[[row["variant"], false]] = row["participated"].to_i - row["converted"].to_i
+        end
+      else
+        data = relation.group(:converted).count
+      end
+
       results = {}
       variants.each do |variant|
         converted = data[[variant, true]].to_i
@@ -127,6 +166,14 @@ module FieldTest
       !winner
     end
 
+    def use_events?
+      if @use_events.nil?
+        FieldTest.events_supported?
+      else
+        @use_events
+      end
+    end
+
     def self.find(id)
       experiment = all.index_by(&:id)[id.to_s]
       raise FieldTest::ExperimentNotFound unless experiment

data/lib/field_test/helpers.rb

@@ -28,7 +28,7 @@ module FieldTest
 
       participants = field_test_participants(options)
 
-      exp.convert(participants)
+      exp.convert(participants, goal: options[:goal])
     end
 
     def field_test_experiments(options = {})

data/lib/field_test/version.rb

@@ -1,3 +1,3 @@
 module FieldTest
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/generators/field_test/events_generator.rb

@@ -0,0 +1,27 @@
+require "rails/generators"
+require "rails/generators/migration"
+require "active_record"
+require "rails/generators/active_record"
+
+module FieldTest
+  module Generators
+    class EventsGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      source_root File.expand_path("../templates", __FILE__)
+
+      # Implement the required interface for Rails::Generators::Migration.
+      def self.next_migration_number(dirname) #:nodoc:
+        next_migration_number = current_migration_number(dirname) + 1
+        if ::ActiveRecord::Base.timestamped_migrations
+          [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
+        else
+          "%.3d" % next_migration_number
+        end
+      end
+
+      def copy_migration
+        migration_template "events.rb", "db/migrate/create_field_test_events.rb"
+      end
+    end
+  end
+end

data/lib/generators/field_test/templates/events.rb

@@ -0,0 +1,11 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def change
+    create_table :field_test_events do |t|
+      t.integer :field_test_membership_id
+      t.string :name
+      t.timestamp :created_at
+    end
+
+    add_index :field_test_events, :field_test_membership_id
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: field_test
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -125,6 +125,7 @@ files:
 - app/controllers/field_test/experiments_controller.rb
 - app/controllers/field_test/memberships_controller.rb
 - app/controllers/field_test/participants_controller.rb
+- app/models/field_test/event.rb
 - app/models/field_test/membership.rb
 - app/views/field_test/experiments/_experiments.html.erb
 - app/views/field_test/experiments/index.html.erb
@@ -140,8 +141,10 @@ files:
 - lib/field_test/helpers.rb
 - lib/field_test/participant.rb
 - lib/field_test/version.rb
+- lib/generators/field_test/events_generator.rb
 - lib/generators/field_test/install_generator.rb
 - lib/generators/field_test/templates/config.yml
+- lib/generators/field_test/templates/events.rb
 - lib/generators/field_test/templates/memberships.rb
 homepage: https://github.com/ankane/field_test
 licenses: []