a_b 0.1.0

data/lib/a_b/controller/sessions.rb

@@ -0,0 +1,28 @@
+Application.class_eval do
+  
+  get '/sessions/new' do
+    @css = %w(log_in)
+    @error = flash[:error]
+    haml :log_in, :layout => :layout
+  end
+  
+  post '/sessions/create' do
+    @user_session = UserSession.new(params[:session])
+    if @user_session.save
+      redirect '/'
+    else
+      flash[:error] = "Log in failed."
+      redirect '/sessions/new'
+    end
+  end
+  
+  get '/sessions/destroy' do
+    current_user_session.destroy
+    redirect '/sessions/new'
+  end
+  
+  get '/css/log_in.css' do
+    headers 'Content-Type' => 'text/css; charset=utf-8'
+    sass :log_in
+  end
+end

data/lib/a_b/controller/tests.rb

@@ -0,0 +1,37 @@
+Application.class_eval do
+  
+  get '/tests/:id/destroy' do
+    restrict
+    @test = ABTest.find params[:id]
+    if @test
+      @test.destroy
+      flash[:success] = 'Test deleted successfully.'
+    else
+      flash[:error] = 'Could not delete test.'
+    end
+    redirect '/'
+  end
+  
+  post '/tests/:id/update' do
+    restrict
+    @test = ABTest.find params[:id]
+    if @test
+      @test.update_attributes params[:test]
+      flash[:success] = 'Test updated successfully.'
+    else
+      flash[:error] = 'Could not update test.'
+    end
+    redirect '/'
+  end
+  
+  post '/tests/create' do
+    restrict
+    @test = ABTest.create params[:test]
+    if @test.id
+      flash[:success] = 'Test created successfully.'
+    else
+      flash[:error] = 'Could not create test.'
+    end
+    redirect '/'
+  end
+end

data/lib/a_b/controller/variants.rb

@@ -0,0 +1,14 @@
+Application.class_eval do
+  
+  get '/variants/:id/destroy' do
+    restricts
+    @variant = ABVariant.find params[:id]
+    if @variant
+      @variant.destroy
+      flash[:success] = 'Variant deleted successfully.'
+    else
+      flash[:error] = 'Could not delete variant.'
+    end
+    redirect '/'
+  end
+end

data/lib/a_b/helper/api.rb

@@ -0,0 +1,8 @@
+Application.class_eval do
+  helpers do
+    
+    def valid_token?
+      Digest::SHA256.hexdigest(params[:session_id] + Token.cached) == params[:token]
+    end
+  end
+end

data/lib/a_b/helper/application.rb

@@ -0,0 +1,56 @@
+Application.class_eval do
+  helpers do
+    
+    # Assets
+    
+    def css(name, media='screen, projection')
+      options = {
+        :href => "/css/#{name}.css",
+        :media => "#{media}",
+        :rel => "stylesheet",
+        :type => "text/css"
+      }
+      haml "%link#{options.inspect}", :layout => false
+    end
+    
+    def javascripts(&block)
+      (@js || []).each do |path|
+        block.call path
+      end
+    end
+    
+    def js(name)
+      options = {
+        :type => "text/javascript",
+        :src => "/js/#{name}.js"
+      }
+      haml "%script#{options.inspect}", :layout => false
+    end
+    
+    def partial(name, options={})
+      haml name, options.merge(:layout => false)
+    end
+    
+    def stylesheets(&block)
+      (@css || []).each do |path|
+        block.call path
+      end
+    end
+    
+    # Authentication
+    
+    def current_user_session
+      return @current_user_session if defined?(@current_user_session)
+      @current_user_session = UserSession.find
+    end
+    
+    def current_user
+      return @current_user if defined?(@current_user)
+      @current_user = current_user_session && current_user_session.record
+    end
+    
+    def restrict
+      redirect '/sessions/new' unless current_user
+    end
+  end
+end

data/lib/a_b/helper/index.rb

@@ -0,0 +1,12 @@
+Application.class_eval do
+  helpers do
+    
+    def table_class(test, index)
+      (index + 1) % 3 == 0 || @tests.last == test ? 'last' : nil
+    end
+    
+    def table_json(test)
+      { :name => test.name, :ticket_url => test.ticket_url, :variant_names => test.variant_names }.to_json
+    end
+  end
+end

data/lib/a_b/model/a_b_test.rb

@@ -0,0 +1,65 @@
+class ABTest < ActiveRecord::Base
+  
+  set_table_name :a_b_tests
+  has_many :variants, :class_name => 'ABVariant', :foreign_key => 'a_b_test_id', :dependent => :delete_all
+  
+  validates_uniqueness_of :name
+  
+  after_save :destroy_variants
+  after_save :create_variants
+  after_save :set_control
+  
+  def control
+    self.variants.find_by_control true
+  end
+  
+  def sorted_variants
+    self.variants.find(:all, :order => 'control desc, conversions / visitors desc, visitors desc')
+  end
+  
+  def variant_names
+    names = self.variants.collect &:name
+    names.delete self.control.name
+    names.unshift self.control.name
+  end
+  
+  def variants=(names)
+    names = names.gsub(/[^,\w]/, '').split(',').uniq
+    current_names = variants.collect(&:name).uniq
+    @control = names.first
+    @new_variant_names = names - current_names
+    @removed_variant_names = current_names - names
+  end
+  
+  private
+  
+  def create_variants
+    return unless @new_variant_names
+    @new_variant_names.each do |name|
+      self.variants.create(:name => name)
+    end
+  end
+  
+  def destroy_variants
+    return unless @removed_variant_names
+    remove = self.variants.find(:all, :conditions => { :name => @removed_variant_names })
+    remove.each do |variant|
+      variant.destroy
+    end
+  end
+  
+  def set_control
+    if @control
+      if self.control
+        self.control.update_attribute(:control, false)
+      end
+      if name = self.variants.find_by_name(@control)
+        name.update_attribute(:control, true)
+      elsif !self.variants.empty?
+        self.variants.first.update_attribute(:control, true)
+      end
+    elsif !self.control && !self.variants.empty?
+      self.variants.first.update_attribute(:control, true)
+    end
+  end
+end

data/lib/a_b/model/a_b_variant.rb

@@ -0,0 +1,131 @@
+class ABVariant < ActiveRecord::Base
+  
+  set_table_name :a_b_variants
+  belongs_to :test, :class_name => 'ABTest', :foreign_key => 'a_b_test_id'
+  
+  after_destroy :save_test
+  
+  validates_uniqueness_of :name
+  
+  def confidence
+    if compute_confidence?
+      cumulative_normal_distribution(z_score(self.test.control))
+    else
+      'n/a'
+    end
+  end
+  
+  def confidence_ok?
+    cumulative_normal_distribution(z_score(self.test.control)) >= 0.95
+  end
+  
+  def conversion_rate
+    if conversions > 0
+      1.0 * conversions / visitors
+    else
+      0.0
+    end
+  end
+  
+  def conversion_rate_ok?
+    conversion_rate > self.test.control.conversion_rate
+  end
+  
+  def pretty_confidence
+    pretty confidence
+  end
+  
+  def pretty_conversion_rate
+    pretty conversion_rate
+  end
+  
+  def suggested_visitors
+    size = sample_size(self.test.control)
+    if conversion_rate == 0 || size < 100
+      100
+    else
+      commafy size
+    end
+  end
+  
+  def suggested_visitors_ok?
+    visitors > sample_size(self.test.control)
+  end
+  
+  def suggested_visitors_with_commas
+    commafy suggested_visitors
+  end
+  
+  def visitors_with_commas
+    commafy visitors
+  end
+  
+  private
+  
+  def commafy(num)
+    num.to_s.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2')
+  end
+  
+  def cumulative_normal_distribution(z)
+    b1 =  0.319381530
+    b2 = -0.356563782
+    b3 =  1.781477937
+    b4 = -1.821255978
+    b5 =  1.330274429
+    p  =  0.2316419
+    c  =  0.39894228
+
+    if z >= 0.0
+      t = 1.0 / (1.0 + p * z)
+      (1.0 - c * Math.exp(-z * z / 2.0) * t * (t * (t * (t * (t * b5 + b4) + b3) + b2) + b1))
+    else
+      t = 1.0 / (1.0 - p * z)
+      (c * Math.exp(-z * z / 2.0) * t * (t * (t * (t * (t * b5 + b4) + b3) + b2) + b1))
+    end
+  end
+  
+  def compute_confidence?
+    self.test &&
+    self.test.control && self != self.test.control &&
+    self.visitors > 0
+  end
+  
+  def pretty(num)
+    if num.respond_to?(:strip)
+      num
+    else
+      sprintf("%.3f", num)[1..-1]
+    end
+  end
+  
+  def sample_size(control)
+    # confidence level is 95% and the desired power is 80%
+    variance = control.conversion_rate * (1 - control.conversion_rate)
+    sensitivity = (self.conversion_rate - control.conversion_rate) ** 2
+    if sensitivity != 0
+      (16 * variance / sensitivity).to_i
+    else
+      0
+    end
+  end
+  
+  def save_test
+    self.test.save if self.test
+  end
+  
+  def z_score(control)
+    cr1 = control.conversion_rate
+    cr2 = self.conversion_rate
+
+    v1 = control.visitors
+    v2 = self.visitors
+    
+    if v1 == 0.0 || v2 == 0.0
+      0.0
+    else
+      z = cr2 - cr1
+      s = (cr2 * (1 - cr2)) / v2 + (cr1 * (1 - cr1)) / v1
+      z / Math.sqrt(s)
+    end
+  end
+end

data/lib/a_b/model/token.rb

@@ -0,0 +1,22 @@
+class Token < ActiveRecord::Base
+  
+  set_table_name :a_b_tokens
+  @@cached_at = nil
+  
+  def self.cached
+    if @@cached_at.nil? || @@cached_at < Time.now.utc - 60
+      @@cached = self.last.token
+      @@cached_at = Time.now.utc
+    end
+    @@cached
+  end
+  
+  def self.generate!
+    if !self.last or (self.last.created_at < Time.now.utc - 60 * 60)
+      token = self.create(:token => Authlogic::Random.friendly_token).token
+    else
+      token = self.last.token
+    end
+    token
+  end
+end

data/lib/a_b/model/user.rb

@@ -0,0 +1,5 @@
+class User < ActiveRecord::Base
+  
+  set_table_name :a_b_users
+  acts_as_authentic
+end

data/lib/a_b/model/user_session.rb

@@ -0,0 +1,4 @@
+class UserSession < Authlogic::Session::Base
+  params_key 'token'
+  single_access_allowed_request_types :any
+end

data/lib/a_b/view/index.haml

@@ -0,0 +1,129 @@
+%hr.space
+%hr.space
+
+.span-16
+  %h2{ :class => @tests.empty? ? 'hide' : nil }
+    %a{ :href => '#' } New A/B Test
+  .span-15.first
+    #new_test.box{ :class => @tests.empty? ? nil : 'hide' }
+      %form{ :action => '/tests/create', :method => 'post', :class => 'clearfix' }
+        .span-7
+          %label{ :for => 'test[name]' } Test name
+          %br/
+          %input{ :type => 'text', :name => 'test[name]' }
+        .span-7.last
+          %label{ :for => 'test[ticket_url]' } Ticket URL
+          %br/
+          %input{ :type => 'text', :name => 'test[ticket_url]' }
+        %hr.space
+        .span-10
+          %label{ :for => 'test[variants]' }
+            Variant names
+            (<a class="tooltip" title="variant_names">more info</a>)
+          %br/
+          %input{ :type => 'text', :name => 'test[variants]' }
+        .buttons.span-4.last
+          %input{ :type => 'submit', :value => 'Create', :class => 'submit' }
+          %input{ :type => 'submit', :value => 'Cancel', :class => 'cancel' }
+
+#search.span-8.last
+  - unless @tests.empty?
+    %form
+      %input.span-5{ :type => 'text' }/
+      %input{ :type => 'submit', :value => 'Search' }
+
+- if @success || @error
+  .span-23.first
+    - if @success
+      .success= @success
+    - if @error
+      .error= @error
+
+%hr.space
+
+- @tests.each_with_index do |test, index|
+  .table.span-8{ :class => table_class(test, index) }[test]
+    .data.hide
+      = table_json(test)
+    .title.clearfix
+      .span-5.search
+        %h3= test.name
+      .span-1.ticket
+        %a{ :href => test.ticket_url } Ticket
+      .span-1.edit
+        %a{ :href => '#' } Edit
+      .span-1.last x
+    .rows
+      .header.clearfix
+        .span-2 Name
+        .span-3.tooltip{ :title => 'visitors_suggested' } Visitors/Suggested
+        .span-1.tooltip{ :title => 'conf' } Conf
+        .span-1.last.tooltip{ :title => 'rate' } Rate
+      - test.sorted_variants.each do |variant|
+        .row.clearfix.search[variant]
+          .span-2
+            - if variant.control?
+              %strong= variant.name
+            - else
+              = variant.name
+          .span-3
+            %span{ :class => variant.suggested_visitors_ok? ? 'green' : nil }
+              = variant.visitors_with_commas
+              \/
+              = variant.suggested_visitors_with_commas
+          .span-1
+            %span{ :class => variant.confidence_ok? ? 'green' : nil }
+              = variant.pretty_confidence
+          .span-1
+            %span{ :class => variant.conversion_rate_ok? ? 'green' : nil }
+              = variant.pretty_conversion_rate
+          .span-1.last x
+  - if table_class(test, index)
+    %hr.space
+    %hr.space
+
+%textarea.hide#tooltip_variant_names
+  %b Variant names
+  %ul
+    %li This field is a comma separated list of variant names.
+    %li The first variant is the control.
+    %li Variant names must be unique across all tests.
+    %li Variant names must only consist of letters, numbers, and underscores.
+
+%textarea.hide#tooltip_visitors_suggested
+  %b Visitors
+  Number of exposures for this variant.
+  %hr.space
+  %b Suggested Visitors
+  Number of exposures necessary to trust that the amount of change between the control and the
+  variant is significantly different.
+
+%textarea.hide#tooltip_conf
+  %b Confidence
+  A variant is statistically better than the control if confidence is greater than or equal to 0.95.
+
+%textarea.hide#tooltip_rate
+  %b Conversion Rate
+  The percentage of users who converted in decimal form.
+
+%textarea.hide#edit_template
+  .span-7.first.edit_container
+    .box.clearfix
+      %form{ :action => '/tests/:id/update', :method => 'post' }
+        %p
+          %label{ :for => 'test[name]' } Test name
+          %br/
+          %input{ :type => 'text', :name => 'test[name]' }
+        %p
+          %label{ :for => 'test[ticket_url]' } Ticket URL
+          %br/
+          %input{ :type => 'text', :name => 'test[ticket_url]' }
+        %p
+          %label{ :for => 'test[variants]' }
+            Variant names
+            (<a class="tooltip" title="variant_names">more info</a>)
+          %br/
+          %input{ :type => 'text', :name => 'test[variants]' }
+        .buttons
+          %input{ :type => 'submit', :value => 'Update', :class => 'submit' }
+          %input{ :type => 'submit', :value => 'Cancel', :class => 'cancel' }