ablab 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f0cb7c5be15bedc01af115b00e2204722b62c16
-  data.tar.gz: d709ec056777764497723f456096350f85ca7260
+  metadata.gz: 82b09b75abb045ba604cc9940801370381565a4f
+  data.tar.gz: 7687acf2d7bf62c7027eea04b411f143b6651093
 SHA512:
-  metadata.gz: 61bd3520e7d2801faec29ca4835d5a6e0ba89d9417b4326c098acf57f68614149780f9957f1fa8de2c500697f6c353c44fdbff4baec6ab300b8b3969ac0284f0
-  data.tar.gz: 19284473fcd4e90840acae8786207de33869f93fb2155b0ba75ac94b53b897a484f6b9f31340b8e6e23489e04b5a5f482e181b39d2c3e49805bf9db0a82a607d
+  metadata.gz: dd0c52bec7918a4f6c783311647fc59e6f92b217c0807474036e97a173679e7d29e2d6b419c3de87c0c61b0809dcb0260500e64200dfc330d23573ec20778ca0
+  data.tar.gz: 0ca01138cc6a81182efc7cd0bf862141fbd68baba8e516b10400315f9fe0088fcd54ef3a4d75570d99e1b10d2f3ee132600da5c06caf81d629a921450ba3f557

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 Luca Ongaro
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,6 +1,24 @@
-require "bundler/gem_tasks"
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Ablab'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+load 'rails/tasks/statistics.rake'
+
 require "rspec/core/rake_task"
 
-RSpec::Core::RakeTask.new(:spec)
+Bundler::GemHelper.install_tasks
 
-task :default => :spec
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/app/assets/javascripts/ablab/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/javascripts/ablab/tracker.js

@@ -0,0 +1,19 @@
+var Ablab = (function(scripts) {
+  var currentScript = scripts[scripts.length - 1];
+  return {
+    debug: false,
+    baseURL: '/ablab',
+    trackView: function(experiment) {
+      this._track(experiment, 'view');
+    },
+    trackSuccess: function(experiment) {
+      this._track(experiment, 'success');
+    },
+    _track: function(experiment, evt) {
+      var trackerPath = (this.baseURL) + '/track.js';
+      var script = document.createElement('script');
+      script.setAttribute('src', trackerPath + '?experiment=' + experiment + '&event=' + evt);
+      currentScript.parentNode.appendChild(script);
+    }
+  }
+})(document.getElementsByTagName("script"));

data/app/assets/stylesheets/ablab/ablab.css

@@ -0,0 +1,99 @@
+/*
+  Place all the styles related to the matching controller here.
+  They will automatically be included in application.css.
+*/
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  font-family: sans-serif;
+  background: #f5f5f5;
+  color: #555555;
+  font-size: 15px;
+  padding: 0;
+  margin: 0;
+}
+
+body > * {
+  padding: 1em 1.618em;
+}
+
+h1, h2, h3 {
+  margin: 0 0 0.618em 0;
+}
+
+header {
+  background: #777777;
+  border-bottom: 2px solid #555555;
+}
+
+header h1 {
+  color: #c4c4c4;
+  text-shadow: 0 -1px 0 #555555;
+  margin: 0;
+  font-size: 24px;
+}
+
+p {
+  margin: 0 0 0.618em 0;
+}
+
+.experiment {
+  padding-top: 1.618em;
+  padding-bottom: 2.5em;
+  border-bottom: 1px solid #ececec;
+  box-shadow: 0 1px 0 #fcfcfc;
+}
+
+.experiment:last-child {
+  border: none;
+  box-shadow: none;
+}
+
+table {
+  border-collapse: collapse;
+  background: #fff4ed;
+}
+
+table tr:nth-child(odd) {
+  background: #ffe9dc;
+}
+
+table tr.control {
+  background: #dddddd;
+}
+
+table tr.control td {
+  border: 1px solid #aaaaaa;
+  color: #555555;
+}
+
+table tr.winner {
+  font-weight: bold;
+  background: #C9F9B5;
+}
+
+table tr.winner td {
+  border: 1px solid #70DC42;
+  color: #2a6212;
+}
+
+table td, table th {
+  color: #652600;
+  padding: 0.618em 1em;
+  border: 1px solid #ffa26b;
+}
+
+table th {
+  background: #ffa26b;
+  border: none;
+  border-bottom: 2px solid #e96e25;
+  border-left: 1px dotted #e96e25;
+}
+
+table {
+  border: 1px solid #ffa26b;
+  box-shadow: 0 1px 2px rgba(0, 0, 0, 0.2);
+}

data/app/assets/stylesheets/ablab/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/ablab/application_controller.rb

@@ -0,0 +1,7 @@
+require 'ablab/helper'
+
+module Ablab
+  class ApplicationController < ActionController::Base
+    include Ablab::Helper
+  end
+end

data/app/controllers/ablab/base_controller.rb

@@ -0,0 +1,25 @@
+require_dependency "ablab/application_controller"
+
+module Ablab
+  class BaseController < ApplicationController
+    if Ablab.dashboard_credentials
+      http_basic_authenticate_with Ablab.dashboard_credentials.merge(only: :dashboard)
+    end
+
+    def track
+      exp = experiment(params[:experiment].to_sym)
+      if params[:event] == 'success'
+        exp.track_success!
+      else
+        exp.track_view!
+      end
+      respond_to do |format|
+        format.js
+      end
+    end
+
+    def dashboard
+      @experiments = Ablab.experiments
+    end
+  end
+end

data/app/helpers/ablab/ablab_helper.rb

@@ -0,0 +1,46 @@
+module Ablab
+  module AblabHelper
+    def format_ratio(numerator, denominator, na = 'n/a')
+      return na if denominator == 0
+      '%.3f' % (numerator.to_f / denominator)
+    end
+
+    def format_number(number, na = 'n/a')
+      return na if number.nil? || number.try(:nan?)
+      number_with_delimiter(number, separator: '.', delimiter: ',')
+    end
+
+    def format_decimal(decimal, na = 'n/a')
+      return na if decimal.nil? || decimal.nan?
+      '%.3f' % decimal
+    end
+
+    def significant?(z_score)
+      z_score && z_score >= 1.65
+    end
+
+    def confidence(z_score, na = 'n/a')
+      return na if z_score.nil? || z_score.nan?
+      if z_score >= 2.58
+        '99%'
+      elsif z_score >= 1.96
+        '95%'
+      elsif z_score >= 1.65
+        '90%'
+      else
+        'insufficient'
+      end
+    end
+
+    def winner?(experiment, group_name)
+      winner_name, winner_results = experiment.results.max_by do |(_, r)|
+        if r[:sessions] > 0
+          r[:conversions].to_f / r[:sessions]
+        else
+          -1
+        end
+      end
+      significant?(winner_results[:z_score]) && winner_name == group_name
+    end
+  end
+end

data/app/helpers/ablab/application_helper.rb

@@ -0,0 +1,4 @@
+module Ablab
+  module ApplicationHelper
+  end
+end

data/app/views/ablab/base/dashboard.html.erb

@@ -0,0 +1,39 @@
+<header>
+  <h1>A/B Tests Dashboard</h1>
+</header>
+
+<% @experiments.each do |name, experiment| %>
+  <div class="experiment">
+    <h2><%= experiment.name.to_s.titleize %></h2>
+    <p><%= experiment.description %></p>
+    <% if experiment.goal %>
+      <p><b>Goal:</b> <%= experiment.goal %></p>
+    <% end %>
+    <table>
+      <tr>
+        <th>Group</th>
+        <th>Description</th>
+        <th>Views</th>
+        <th>Sessions</th>
+        <th>Successes</th>
+        <th>Conversions</th>
+        <th>Conversion rate</th>
+        <th>Z-score</th>
+        <th>Confidence</th>
+      </tr>
+      <% experiment.results.each do |group_name, results| %>
+        <tr class="<%= 'winner' if winner?(experiment, group_name) %> <%= 'control' if results[:control] %>">
+          <td title="<%= results[:description] %>"><b><%= group_name.to_s.titleize %></b></td>
+          <td><%= results[:description] %></td>
+          <td><%= format_number results[:views] %></td>
+          <td><%= format_number results[:sessions] %></td>
+          <td><%= format_number results[:successes] %></td>
+          <td><%= format_number results[:conversions] %></td>
+          <td><%= format_ratio results[:conversions].to_f, results[:sessions] %></td>
+          <td><%= format_decimal results[:z_score] %></td>
+          <td><%= confidence results[:z_score] %></td>
+        </tr>
+      <% end %>
+    </table>
+  </div>
+<% end %>

data/app/views/ablab/base/track.js.erb

@@ -0,0 +1,3 @@
+if (console && console.log && Ablab && Ablab.debug) {
+  console.log('Ablab: tracked <%= params[:event] %> on <%= params[:experiment] %>');
+}

data/app/views/layouts/ablab/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Ablab</title>
+  <%= stylesheet_link_tag    "ablab/application", media: "all" %>
+  <%= javascript_include_tag "ablab/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,5 @@
+Ablab::Engine.routes.draw do
+  get '/track', to: 'base#track'
+
+  root to: 'base#dashboard'
+end

data/lib/ablab.rb

@@ -1,8 +1,10 @@
 require "ablab/version"
-require "ablab/controller"
+require "ablab/helper"
 require "ablab/store"
+require "ablab/engine"
+require "forwardable"
 
-module ABLab
+module Ablab
   module ModuleMethods
     attr_reader :experiments
 
@@ -20,23 +22,36 @@ module ABLab
         @tracker = Class.new(*args)
       else
         class_name = type.to_s.split('_').map(&:capitalize).join
-        @tracker = ABLab::Store.const_get(class_name).new(*args)
+        @tracker = Ablab::Store.const_get(class_name).new(*args)
       end
     end
 
     def tracker
-      @tracker ||= ABLab::Store::Memory.new
+      @tracker ||= Ablab::Store::Memory.new
+    end
+
+    def dashboard_credentials(credentials = nil)
+      if credentials
+        unless credentials[:name] && credentials[:password]
+          raise InvalidCredentials, 'credentials should provide name and password'
+        end
+        @dashboard_credentials = credentials
+      end
+      @dashboard_credentials
     end
   end
 
+  class InvalidCredentials < ArgumentError; end
+
   extend ModuleMethods
 
   class Experiment
     attr_reader :name, :groups, :control
 
     def initialize(name, &block)
-      @name   = name
-      @groups = []
+      @name    = name.to_sym
+      @control = Group.new(:control, 'control group')
+      @groups  = [@control]
       instance_exec(&block)
     end
 
@@ -45,9 +60,13 @@ module ABLab
       @description
     end
 
+    def goal(goal = nil)
+      @goal = goal if goal
+      @goal
+    end
+
     def group(name, options = {})
       group = Group.new(name, options[:description])
-      @control = group if options[:control]
       @groups << group
     end
 
@@ -56,19 +75,17 @@ module ABLab
       @result.data
     end
 
-    def run(uid)
-      draw = Random.new(uid.hash * name.hash).rand(1000)
-      Run.new(self, draw)
+    def run(session_id)
+      Run.new(self, session_id)
     end
   end
 
   class Run
-    attr_reader :group, :experiment
+    attr_reader :experiment, :session_id
 
-    def initialize(experiment, draw)
-      idx         = (draw / (1000.0 / experiment.groups.size)).floor
+    def initialize(experiment, session_id)
+      @experiment, @session_id = experiment, session_id
       @experiment = experiment
-      @group      = experiment.groups[idx].name
     end
 
     def in_group?(name)
@@ -76,15 +93,30 @@ module ABLab
     end
 
     def track_view!
-      ABLab.tracker.track_view!(experiment.name, group)
+      Ablab.tracker.track_view!(experiment.name, group, session_id)
     end
 
-    def track_conversion!
-      ABLab.tracker.track_conversion!(experiment.name, group)
+    def track_success!
+      Ablab.tracker.track_success!(experiment.name, group, session_id)
+    end
+
+    def group
+      return @group unless @group.nil?
+      idx = (draw / (1000.0 / experiment.groups.size)).floor
+      @group = experiment.groups[idx].name
+    end
+
+    def draw
+      Random.new(session_id.hash ^ experiment.name.hash).rand(1000)
     end
   end
 
-  class Group < Struct.new(:name, :description); end
+  class Group
+    attr_reader :name, :description
+    def initialize(name, description = nil)
+      @name, @description = name.to_sym, description
+    end
+  end
 
   class Result
     extend Forwardable
@@ -95,30 +127,27 @@ module ABLab
     end
 
     def data
-      raise NoControlGroup.new("no control group") if control.nil?
-      c_views, c_conv = views_and_conversions(control)
+      counts_c = counts(control)
       groups.map do |group|
         if group == control
-          next { views: c_views, conversions: c_conv, control: true }
+          next [group.name, counts_c.merge(control: true, description: group.description)]
         end
-        views, conv = views_and_conversions(group)
-        z = z_score(views, conv, c_views, c_conv)
-        { views: views, conversions: conv, z_score: z, control: false }
-      end
+        counts = counts(group)
+        z = z_score(counts[:sessions], counts[:conversions],
+                    counts_c[:sessions], counts_c[:conversions])
+        [group.name, counts.merge(z_score: z, control: false, description: group.description)]
+      end.to_h
     end
 
-    private def views_and_conversions(group)
-      views       = ABLab.tracker.views(name, group.name)
-      conversions = ABLab.tracker.conversions(name, group.name)
-      [views, conversions]
+    private def counts(group)
+      Ablab.tracker.counts(name, group.name)
     end
 
-    private def z_score(views, conv, c_views, c_conv)
-      p  = conv.to_f / views
-      pc = c_conv.to_f / c_views
-      (p - pc) / Math.sqrt((p*(1 - p) / views) + (pc*(1 - pc) / c_views))
+    private def z_score(s, c, sc, cc)
+      return nil if s == 0 || sc == 0
+      p  = c.to_f / s
+      pc = cc.to_f / sc
+      (p - pc) / Math.sqrt((p*(1 - p) / s) + (pc*(1 - pc) / sc))
     end
-
-    class NoControlGroup < StandardError; end
   end
 end