trailguide 0.1.31 → 0.2.0

data/app/views/trail_guide/admin/experiments/show.html.erb

@@ -0,0 +1,78 @@
+<div class="experiment row justify-content-center">
+  <div class="col-sm-12 col-md-10 col-lg-8">
+    <%= render partial: "header", locals: { experiment: @experiment } %>
+
+    <% if @experiment.combined? %>
+      <table class="table table-hover">
+        <% combined_experiments = @experiment.combined.map { |e| TrailGuide.catalog.find(e) } %>
+        <% combined_experiments.each do |experiment| %>
+          <%= render partial: 'thead', locals: { experiment: experiment } %>
+          <%= render partial: 'tbody', locals: { experiment: experiment, calculator: experiment_calculator(experiment) } %>
+        <% end %>
+
+        <tfoot class="thead-light">
+          <tr>
+            <th class="btn-col">&nbsp;</th>
+            <th scope="row">&nbsp;</th>
+            <th>
+              <span><%= number_with_delimiter @experiment.participants %></span>
+              <% if @experiment.configuration.target_sample_size %>
+                <span class="text-muted">/ <%= number_with_delimiter @experiment.configuration.target_sample_size %></span>
+              <% end %>
+            </th>
+            <% if @experiment.goals.empty? %>
+              <th><%= experiment_metric @experiment, combined_experiments.sum(&:converted) %></th>
+            <% else %>
+              <% @experiment.goals.each do |goal| %>
+                <th><%= experiment_metric @experiment, combined_experiments.sum { |e| e.converted(goal) } %></th>
+              <% end %>
+            <% end %>
+            <th class="btn-col">&nbsp;</th>
+          </tr>
+        </tfoot>
+      </table>
+    <% else %>
+      <table class="table table-hover">
+        <%= render partial: 'thead', locals: { experiment: @experiment } %>
+        <%= render partial: 'tbody', locals: { experiment: @experiment, calculator: experiment_calculator(@experiment) } %>
+
+        <tfoot class="thead-light">
+          <tr>
+            <th class="btn-col">&nbsp;</th>
+            <th scope="row">&nbsp;</th>
+            <th>
+              <span><%= number_with_delimiter @experiment.participants %></span>
+              <% if @experiment.configuration.target_sample_size %>
+                <span class="text-muted">/ <%= number_with_delimiter @experiment.configuration.target_sample_size %></span>
+              <% end %>
+            </th>
+            <% if @experiment.goals.empty? %>
+              <th><%= experiment_metric @experiment, @experiment.converted %></th>
+            <% else %>
+              <% @experiment.goals.each do |goal| %>
+                <th><%= experiment_metric @experiment, @experiment.converted(goal) %></th>
+              <% end %>
+            <% end %>
+            <th class="btn-col">&nbsp;</th>
+          </tr>
+        </tfoot>
+      </table>
+    <% end %>
+
+    <div class="row" style="display: none;">
+      <div class="col-12">
+        <% if @analyzing && experiment_metrics_visible?(@experiment) %>
+          <div class="alert alert-primary">
+            <h6>
+              <span class="fas fa-chart-line"></span>
+              Analysis
+            </h6>
+            <p><small>TODO</small></p>
+          </div>
+        <% end %>
+      </div>
+    </div>
+
+    <%= render partial: 'alert_peek', locals: {experiment: @experiment} %>
+  </div>
+</div>

data/app/views/trail_guide/admin/groups/index.html.erb

@@ -0,0 +1,40 @@
+<%= render partial: 'trail_guide/admin/orphans/alert' %>
+
+<div class="row justify-content-center">
+  <div class="col-12 col-md-10 col-lg-8">
+    <div class="experiments">
+      <% @groups.each do |group| %>
+        <button href="<%= trail_guide_admin.group_path(group) %>" class="experiment btn btn-outline-secondary rounded-lg text-left">
+          <div class="row align-items-center">
+            <div class="col-12 col-md-8 col-lg-9 experiment-text">
+              <h4 id="<%= group %>">
+                <%= group.to_s.humanize.titleize %>
+              </h4>
+            </div>
+            <div class="col-12 col-md-4 col-lg-3 text-center text-md-right">
+              <div class="row align-items-center">
+                <div class="col-6 col-md-7 col-lg-8 experiment-text text-center">
+                  <h4>
+                    <span class="fas fa-flask"></span>
+                    &times;
+                    <%= TrailGuide.catalog.select(group).count %>
+                  </h4>
+                </div>
+
+                <div class="col-6 col-md-5 col-lg-4 experiment-text text-center">
+                  <h4>
+                    <% if TrailGuide.catalog.select(group).all? { |e| e.goals.map(&:name).include?(group) } %>
+                      <span class="fas fa-fill" data-toggle="tooltip" title="this group represents a shared conversion goal between multiple experiments"></span>
+                    <% else %>
+                      <span class="fas fa-th-large" data-toggle="tooltip" title="this group is not linked to any shared goals, and is purely organizational"></span>
+                    <% end %>
+                  </h4>
+                </div>
+              </div>
+            </div>
+          </div>
+        </button>
+      <% end %>
+    </div>
+  </div>
+</div>

data/app/views/trail_guide/admin/groups/show.html.erb

@@ -0,0 +1,9 @@
+<div class="row justify-content-center">
+  <div class="col-12 col-md-10 col-lg-8">
+    <div class="experiments">
+      <% @experiments.each do |experiment| %>
+        <%= render 'trail_guide/admin/experiments/experiment', experiment: experiment %>
+      <% end %>
+    </div>
+  </div>
+</div>

data/app/views/trail_guide/admin/orphans/_alert.html.erb

@@ -0,0 +1,44 @@
+<% unless TrailGuide.configuration.ignore_orphaned_groups? || TrailGuide.catalog.orphans.empty? %>
+  <div class="row justify-content-center">
+    <div class="col-sm-12 col-md-10 col-lg-8">
+      <div class="alert alert-warning alert-dismissable">
+        <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+          <span aria-hidden="true">&times;</span>
+        </button>
+
+        <h5>
+          <span class="fas fa-ghost"></span>
+          &nbsp;
+          Orphaned Groups
+        </h5>
+
+        <br />
+        <p>TrailGuide has detected <strong><%= TrailGuide.catalog.orphans.values.sum(&:count) %></strong> calls to <code>trailguide.convert</code> against <strong><%= TrailGuide.catalog.orphans.count %></strong> unknown group <%= "name".pluralize(TrailGuide.catalog.orphans.count) %>.</p>
+        
+        <p><small>This usually means that the last experiment that belonged to a given group was removed without realizing it, and references to the group were overlooked. <strong>Don't worry, your users won't notice a thing</strong>, and you can find all the logged references below. You can find <%= link_to "more info here.", "https://github.com/markrebec/trailguide#orphaned-groups", target: :blank %></small></p>
+
+        <div class="accordion bg-transparent border-0" id="orphans">
+          <% TrailGuide.catalog.orphans.each do |orphan,traces| %>
+            <div class="card">
+                <button class="btn btn-outline-light text-left btn-block" type="button" data-toggle="collapse" data-target="#orphan-<%= orphan %>" aria-expanded="true" aria-controls="collapseOne">
+                  <strong><code><%= orphan %></code></strong>
+                </button>
+
+              <div id="orphan-<%= orphan %>" class="collapse" aria-labelledby="orphan-btn-<%= orphan %>" data-parent="#orphans">
+                <div class="card-body">
+                  <ul style="list-style: none;">
+                    <% traces.each do |trace| %>
+                      <li><code><%= trace %></code></li>
+                    <% end %>
+                  </ul>
+                </div>
+              </div>
+            </div>
+          <% end %>
+        </div>
+
+        <p><small>These warnings will expire after 15 minutes of inactivity, and they will disappear automatically once you remove the references listed above and are no longer being reactivated. If you prefer to clear these warnings immediately after cleaning up, you can run <code>TrailGuide.catalog.adopted(:group_name)</code> from a console, rake task, or background job.</small></p>
+      </div>
+    </div>
+  </div>
+<% end %>

data/config/initializers/trailguide.rb

@@ -17,6 +17,17 @@ TrailGuide.configure do |config|
   # false     you'll need to include the helper module manually where you want
   config.include_helpers = true
 
+  # whether or not to ignore orphaned group/experiment conversion calls in the
+  # admin - this can be useful if you intentionally leave calls to
+  # `trailguide.convert(:group_name)` at key points in your application, but
+  # also periodically add/remove experiments using those groups
+  #
+  # true  will not track orphaned convert calls nor expose them in the admin
+  #       interface
+  # false will track orphaned convert calls, and notify you via the admin
+  #       interface so you can track them down and remove them as you'd like
+  config.ignore_orphaned_groups = false
+
   # request param for overriding/previewing variants - allows previewing
   # variants with request params
   # i.e. example.com/somepage/?experiment[my_experiment]=option_b
@@ -56,11 +67,13 @@ TrailGuide.configure do |config|
   # a couple milliseconds to participant initialization, but isn't strictly
   # necessary, since expired enrollment will never affect future experiment
   # participation - it might be good practice if you're using redis to store
-  # participant data without expiration
+  # participant data without expiration, or to avoid overflowing a client cookie
   #
-  # true      will clean up any old/inactive experiment keys for each
-  #           participant the first time they're encountered during a script
+  # true      will explicitly clean up any old/inactive experiment keys for each
+  #           participant the first time they're initialized during a script
   #           execution (web request, etc.)
+  # :inline   will only clean up any old/inactive experiment keys that are
+  #           encountered when referencing a participant's active experiments
   # false     will skip the cleanup process entirely
   config.cleanup_participant_experiments = true
 
@@ -125,6 +138,19 @@ TrailGuide::Experiment.configure do |config|
   #         the experiment if they encounter it again
   config.reset_manually = true
 
+  # whether or not to store individual participation when returning a variant
+  #
+  # this can be useful if you are using a custom, content-based algorithm where
+  # the variant is determined by content rather than user bucketing, and you
+  # want to treat participation more like impressions (i.e. for seo experiments)
+  #
+  # true    participation is incremented the first time a participant is
+  #         enrolled, and the participant is assigned their selection for future
+  #         reference, stored via the configured participant adapter
+  # false   participation will be incremented every time a variant is returned,
+  #         and the participant will not have their assignment stored
+  config.store_participation = true
+
   # whether or not to enter participants into a variant when using the override
   # parameter to preview variants
   #
@@ -167,6 +193,21 @@ TrailGuide::Experiment.configure do |config|
   # false   prevents converting to multiple goals for a single participant
   config.allow_multiple_goals = false
 
+  # whether or not to enable calibration before an experiment is started - the
+  # participants and conversions will be tracked for your control group while
+  # an experiment remains unstarted, which can be useful for gathering a
+  # baseline conversion rate if you don't already have one
+  #
+  # control is always returned for unstarted experiments by default, and this
+  # configuration only affects whether or not to track metrics
+  #
+  # this setting only applies when start_manually is also true
+  #
+  # true  metrics for participation and conversion will be tracked for the
+  #       control group until the experiment is started
+  # false metrics will not be tracked while the experiment remains unstarted
+  config.enable_calibration = false
+
   # whether or not to skip the request filtering for this experiment - can be
   # useful when defining content-based experiments with custom algorithms which
   # bucket participants strictly based on additional content metadata and you
@@ -208,6 +249,18 @@ TrailGuide::Experiment.configure do |config|
   #
   # config.on_start = -> (experiment, context) { ... }
 
+  # callback on experiment schedule manually via UI/console, can be used for logging,
+  # tracking, etc.
+  #
+  # experiments can only be scheduled if config.start_manually is true
+  #
+  # context may or may not be present depending on how you're triggering the
+  # action - if you're using the admin, this will be the admin controller
+  # context, if you're in a console you have the option to pass a context to
+  # `experiment.schedule!` or not
+  #
+  # config.on_schedule = -> (experiment, start_at, stop_at, context) { ... }
+
   # callback on experiment stop manually via UI/console, can be used for
   # logging, tracking, etc.
   #
@@ -272,26 +325,28 @@ TrailGuide::Experiment.configure do |config|
   # callback when a participant is entered into a variant for the first time,
   # can be used for logging, tracking, etc.
   #
-  # config.on_choose = -> (experiment, variant, metadata) { ... }
+  # config.on_choose = -> (experiment, variant, participant, metadata) { ... }
 
   # callback every time a participant is returned a variant in the experiment,
   # can be used for logging, tracking, etc.
   #
-  # config.on_use = -> (experiment, variant, metadata) { ... }
+  # config.on_use = -> (experiment, variant, participant, metadata) { ... }
 
   # callback when a participant converts for a variant in the experiment, can be
   # used for logging, tracking, etc.
   #
-  # config.on_convert = -> (experiment, variant, checkpoint, metadata) { ... }
+  # config.on_convert = -> (experiment, checkpoint, variant, participant, metadata) { ... }
 
 
   # callback that can short-circuit participation based on your own logic, which
   # gets called *after* all the core engine checks (i.e. that the user is
   # not excluded or already participating, etc.)
   #
+  # `allowed` will be the value returned by any previous callbacks in the chain
+  #
   # should return true or false
   #
-  # config.allow_participation = -> (experiment, metadata) { ... return true }
+  # config.allow_participation = -> (experiment, allowed, participant, metadata) { ... return true }
 
 
   # callback that can short-circuit conversion based on your own logic, which
@@ -299,9 +354,11 @@ TrailGuide::Experiment.configure do |config|
   # participating in the experiment, is within the bounds of the experiment
   # configuration for allow_multiple_*, etc.)
   #
+  # `allowed` will be the value returned by any previous callbacks in the chain
+  #
   # should return true or false
   #
-  # config.allow_conversion = -> (experiment, checkpoint, metadata) { ... return true }
+  # config.allow_conversion = -> (experiment, allowed, checkpoint, variant, participant, metadata) { ... return true }
 
 
   # callback that can be used to modify the rollout of a selected winner - for
@@ -309,9 +366,15 @@ TrailGuide::Experiment.configure do |config|
   # gem to do a "feature rollout" from your control variant to your winner for
   # all users
   #
+  # be aware that when using this alongside track_winner_conversions, whatever
+  # variant is returned from this callback chain is what will be tracked for
+  # participation and conversion as long as the experiment is still running
+  #
+  # `winner` will be the variant returned by any previous callbacks in the chain
+  #
   # must return an experiment variant
   #
-  # config.rollout_winner = -> (experiment, winner) { ... return variant }
+  # config.rollout_winner = -> (experiment, winner, participant) { ... return variant }
 end
 
 # admin ui configuration

data/config/routes/admin.rb

@@ -1,22 +1,29 @@
 TrailGuide::Admin::Engine.routes.draw do
-  resources :experiments, path: '/', only: [:index] do
+  resources :groups, only: [:index, :show]
+
+  resources :experiments, path: '/', only: [:index, :show] do
     member do
-      match :start,   via: [:put, :post, :get]
-      match :pause,   via: [:put, :post, :get]
-      match :stop,    via: [:put, :post, :get]
-      match :reset,   via: [:put, :post, :get]
-      match :resume,  via: [:put, :post, :get]
-      match :restart, via: [:put, :post, :get]
+      match :start,    via: [:put, :post, :get]
+      match :schedule, via: [:put, :post]
+      match :pause,    via: [:put, :post, :get]
+      match :stop,     via: [:put, :post, :get]
+      match :reset,    via: [:put, :post, :get]
+      match :resume,   via: [:put, :post, :get]
+      match :restart,  via: [:put, :post, :get]
 
-      match :join,    via: [:put, :post, :get], path: 'join/:variant'
-      match :leave,   via: [:put, :post, :get]
+      match :enroll,   via: [:put, :post, :get], path: 'enroll'
+      match :join,     via: [:put, :post, :get], path: 'join/:variant'
+      match :convert,  via: [:put, :post, :get], path: 'convert/:goal'
+      match :leave,    via: [:put, :post, :get]
 
-      match :winner,  via: [:put, :post, :get], path: 'winner/:variant'
-      match :clear,   via: [:put, :post, :get]
+      match :winner,   via: [:put, :post, :get], path: 'winner/:variant'
+      match :clear,    via: [:put, :post, :get]
     end
 
     collection do
-      get  '/:scope', action: :index, as: :scoped
+      put  '/import', action: :import, as: :import
+      get  '/export', action: :index,  as: :export, format: [:json]
+      get  '/scope/:scope', action: :index, as: :scoped
     end
   end
 end

data/lib/trail_guide/admin/engine.rb

@@ -6,6 +6,8 @@ module TrailGuide
       config.generators do |g|
         g.test_framework = :rspec
       end
+
+      paths["config/routes.rb"] = "config/routes/admin.rb"
     end
   end
 end

data/lib/trail_guide/admin.rb

@@ -3,5 +3,7 @@ require "trail_guide/admin/engine"
 module TrailGuide
   module Admin
     include Canfig::Module
+
+    DISPLAY_DATE_FORMAT = "%b %e %Y @ %l:%M %p"
   end
 end

data/lib/trail_guide/calculators/bayesian.rb

@@ -0,0 +1,98 @@
+module TrailGuide
+  module Calculators
+    class Bayesian < Calculator
+      def self.enabled?
+        !!(defined?(::Integration) && (defined?(::Rubystats) || defined?(::Distribution)))
+      end
+
+      attr_reader :beta
+
+      def initialize(*args, beta: nil, **opts)
+        raise NoIntegrationLibrary if !defined?(::Integration)
+
+        if beta.nil?
+          # prefer rubystats if not specified
+          if defined?(::Rubystats)
+            beta = :rubystats
+          elsif defined?(::Distribution)
+            beta = :distribution
+          else
+            raise NoBetaDistributionLibrary
+          end
+        end
+
+        case beta.to_sym
+        when :distribution
+          raise NoBetaDistributionLibrary, beta unless defined?(::Distribution)
+          TrailGuide.logger.debug "Using Distribution::Beta to calculate beta distributions"
+          TrailGuide.logger.debug "GSL detected, Distribution::Beta will use GSL for better performance" if defined?(::GSL)
+        when :rubystats
+          raise NoBetaDistributionLibrary, beta unless defined?(::Rubystats)
+          TrailGuide.logger.debug "Using Rubystats::BetaDistribution to calculate beta distributions"
+        else
+          raise UnknownBetaDistributionLibrary, beta
+        end
+
+        super(*args, **opts)
+        @beta = beta.to_sym
+      end
+
+      def pdf(variant, z)
+        x = variant.subset
+        n = variant.superset
+        if beta == :distribution
+          Distribution::Beta.pdf(z, x+1, n-x+1)
+        else
+          Rubystats::BetaDistribution.new(x+1, n-x+1).pdf(z)
+        end
+      end
+
+      def cdf(variant, z)
+        x = variant.subset
+        n = variant.superset
+        if beta == :distribution
+          Distribution::Beta.cdf(z, x+1, n-x+1)
+        else
+          Rubystats::BetaDistribution.new(x+1, n-x+1).cdf(z)
+        end
+      end
+
+      def variant_probability(variant)
+        Integration.integrate(0, 1, tolerance: 1e-4) do |z|
+          vpdf = pdf(variant, z)
+          variants.each do |var|
+            next if var == variant
+            vpdf = vpdf * cdf(var, z)
+          end
+          vpdf
+        end * 100.0
+      end
+
+      def calculate!
+        variants_with_conversion.each do |variant|
+          expvar = experiment.variants.find { |var| var.name == variant.name }
+          vprob = variant_probability(variant)
+          variant.probability = vprob
+          variant.significance = TrailGuide::Calculators::SIGNIFICANT_PROBABILITIES.reverse.find { |pct| vprob >= pct } || 0
+
+          #if worst && variant.measure > worst.measure
+          #  variant.difference = (variant.measure - worst.measure) / worst.measure * 100
+          #end
+          if base
+            if variant.measure > base.measure
+              variant.difference = (variant.measure - base.measure) / base.measure * 100
+            elsif base.measure > variant.measure
+              variant.difference = -((base.measure - variant.measure) / base.measure * 100)
+            else
+              variant.difference = 0
+            end
+          end
+        end
+
+        @choice = best && best.probability >= probability ? best : nil
+
+        self
+      end
+    end
+  end
+end

data/lib/trail_guide/calculators/calculator.rb

@@ -0,0 +1,58 @@
+module TrailGuide
+  module Calculators
+    SIGNIFICANT_PROBABILITIES = [10, 50, 75, 90, 95, 99, 99.9]
+    DEFAULT_PROBABILITY = 90
+
+    class Calculator
+      attr_reader :experiment, :goal, :probability, :choice
+
+      def initialize(experiment, probability=TrailGuide::Calculators::DEFAULT_PROBABILITY, base: :default, goal: nil, against: nil)
+        @experiment = experiment
+        @probability = probability
+        @base_type = base
+        @goal = goal
+        @against = against
+      end
+
+      def variants
+        @variants ||= experiment.variants.map do |variant|
+          superset = @against ? variant.converted(@against) : variant.participants
+          converts = variant.converted(goal)
+          measure = (converts.to_f / superset.to_f) rescue 0
+          measure = 0 if measure.nan?
+
+          Struct.new(:name, :control, :superset, :subset, :measure,
+                     :difference, :probability, :significance, :z_score)
+            .new(variant.name, variant.control?, superset, converts, measure, 0, 0, nil, nil)
+        end.sort_by { |v| v.measure }
+      end
+
+      def variants_with_conversion
+        @variants_with_conversion ||= variants.select { |variant| variant.measure > 0.0 }
+      end
+
+      def base
+        @base ||= case @base_type
+          when :control
+            # use the control as the "base"
+            variants.find { |variant| variant.control }
+          else
+            # use the second-best converting as the "base" (default behavior)
+            variants[-2]
+        end
+      end
+
+      def best
+        @best ||= variants_with_conversion.last
+      end
+
+      def worst
+        @worst ||= variants_with_conversion.first
+      end
+
+      def calculate!
+        raise NotImplementedError, "You must define a calculate! method on your calculator class"
+      end
+    end
+  end
+end

data/lib/trail_guide/calculators/score.rb

@@ -0,0 +1,68 @@
+module TrailGuide
+  module Calculators
+    class Score < Calculator
+      # array of [z-score, percentage]
+      def self.z_score_probabilities
+        @@z_score_probabilities ||= begin
+          avg = 50.0
+          norm_dist = []
+          (0.0..3.1).step(0.01) { |x| norm_dist << [x, avg += 1 / Math.sqrt(2 * Math::PI) * Math::E ** (-x ** 2 / 2)] }
+          norm_dist
+        end
+      end
+
+      def self.all_probabilities
+        @@all_probabilities ||= (0.0..100.0).step(0.1).map { |pct| [z_score_probabilities.find { |x,a| a >= pct }.first, pct] }.reverse
+      end
+
+      def self.significant_probabilities
+        @@significant_probabilities ||= TrailGuide::Calculators::SIGNIFICANT_PROBABILITIES.map { |pct| [z_score_probabilities.find { |x,a| a >= pct }.first, pct] }.reverse
+      end
+
+      def self.z_score_probability(score)
+        score = score.abs
+        probability = all_probabilities.find { |z,p| score >= z }
+        probability ? probability.last : 0
+      end
+
+      def self.significant_probability(score)
+        score = score.abs
+        probability = significant_probabilities.find { |z,p| score >= z }
+        probability ? probability.last : 0
+      end
+
+      def calculate!
+        pc = base.measure
+        nc = base.superset
+        variants_with_conversion.each do |var|
+          p = var.measure
+          n = var.superset
+          z_score = (p - pc) / ((p * (1-p)/n) + (pc * (1-pc)/nc)).abs ** 0.5
+
+          var.z_score = z_score
+          var.probability = self.class.z_score_probability(z_score)
+          var.probability = -(var.probability) if var.z_score.negative?
+          var.significance = self.class.significant_probability(z_score)
+          var.significance = -(var.significance) if var.z_score.negative?
+
+          #if worst && var.measure > worst.measure
+          #  var.difference = (var.measure - worst.measure) / worst.measure * 100
+          #end
+          if base
+            if var.measure > base.measure
+              var.difference = (var.measure - base.measure) / base.measure * 100
+            elsif base.measure > var.measure
+              var.difference = -((base.measure - var.measure) / base.measure * 100)
+            else
+              var.difference = 0
+            end
+          end
+        end
+
+        @choice = best && best.probability >= probability ? best : nil
+
+        self
+      end
+    end
+  end
+end

data/lib/trail_guide/calculators.rb

@@ -0,0 +1,8 @@
+require 'trail_guide/calculators/calculator'
+require 'trail_guide/calculators/score'
+require 'trail_guide/calculators/bayesian'
+
+module TrailGuide
+  module Calculators
+  end
+end