trailguide 0.1.31 → 0.2.0

data/lib/trail_guide/helper.rb

@@ -1,10 +1,10 @@
 module TrailGuide
   module Helper
-    def trailguide(metric=nil, **opts, &block)
+    def trailguide(key=nil, **opts, &block)
       @trailguide_proxy ||= HelperProxy.new(self)
       @trailguide_proxy = HelperProxy.new(self) if @trailguide_proxy.context != self
-      return @trailguide_proxy if metric.nil?
-      @trailguide_proxy.choose!(metric, **opts, &block)
+      return @trailguide_proxy if key.nil?
+      @trailguide_proxy.choose!(key, **opts, &block)
     end
 
     def trailguide_participant
@@ -70,42 +70,42 @@ module TrailGuide
         @participant = participant
       end
 
-      def new(metric)
-        MetricProxy.new(context, metric, participant: participant)
+      def new(key)
+        ExperimentProxy.new(context, key, participant: participant)
       end
 
-      def choose!(metric, **opts, &block)
-        new(metric).choose!(**opts, &block)
+      def choose!(key, **opts, &block)
+        new(key).choose!(**opts, &block)
       end
       alias_method :enroll!, :choose!
 
-      def choose(metric, **opts, &block)
-        new(metric).choose(**opts, &block)
+      def choose(key, **opts, &block)
+        new(key).choose(**opts, &block)
       end
       alias_method :enroll, :choose
 
-      def run!(metric, **opts)
-        new(metric).run!(**opts)
+      def run!(key, **opts)
+        new(key).run!(**opts)
       end
 
-      def run(metric, **opts)
-        new(metric).run(**opts)
+      def run(key, **opts)
+        new(key).run(**opts)
       end
 
-      def render!(metric, **opts)
-        new(metric).render!(**opts)
+      def render!(key, **opts)
+        new(key).render!(**opts)
       end
 
-      def render(metric, **opts)
-        new(metric).render(**opts)
+      def render(key, **opts)
+        new(key).render(**opts)
       end
 
-      def convert!(metric, checkpoint=nil, **opts, &block)
-        new(metric).convert!(checkpoint, **opts, &block)
+      def convert!(key, checkpoint=nil, **opts, &block)
+        new(key).convert!(checkpoint, **opts, &block)
       end
 
-      def convert(metric, checkpoint=nil, **opts, &block)
-        new(metric).convert(checkpoint, **opts, &block)
+      def convert(key, checkpoint=nil, **opts, &block)
+        new(key).convert(checkpoint, **opts, &block)
       end
 
       def participant
@@ -121,17 +121,18 @@ module TrailGuide
       end
     end
 
-    class MetricProxy < HelperProxy
-      attr_reader :metric
+    class ExperimentProxy < HelperProxy
+      attr_reader :key
 
-      def initialize(context, metric, **opts)
+      def initialize(context, key, **opts)
         super(context, **opts)
-        @metric = metric
-        raise NoExperimentsError, "Could not find any experiments matching `#{metric}`." if experiments.empty?
+        @key = key.to_s.underscore.to_sym
       end
 
       def choose!(**opts, &block)
-        raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but the metric `#{metric}` matches more than one experiment." if experiments.length > 1
+        raise NoExperimentsError, key if experiments.empty?
+        raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but `#{key}` matches more than one experiment." if experiments.length > 1
+        raise TooManyExperimentsError, "Selecting a variant requires a single experiment, but `#{key}` refers to a combined experiment." if experiment.combined?
         opts = {override: override_variant, excluded: exclude_visitor?}.merge(opts)
         variant = experiment.choose!(**opts)
         if block_given?
@@ -144,6 +145,8 @@ module TrailGuide
 
       def choose(**opts, &block)
         choose!(**opts, &block)
+      rescue NoExperimentsError => e
+        raise e
       rescue => e
         TrailGuide.logger.error e
         experiment.control
@@ -158,13 +161,13 @@ module TrailGuide
           unless context.respond_to?(varmeth, true)
             if context_type == :controller
               raise NoVariantMethodError,
-                "Undefined local method `#{varmeth}`. You must define a controller method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
+                "Undefined local method `#{varmeth}`. You must define a controller method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
             elsif context_type == :template
               raise NoVariantMethodError,
-                "Undefined local method `#{varmeth}`. You must define a helper method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define ApplicationHelper##{varmeth}(metadata={})"
+                "Undefined local method `#{varmeth}`. You must define a helper method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define ApplicationHelper##{varmeth}(metadata={})"
             else
               raise NoVariantMethodError,
-                "Undefined local method `#{varmeth}`. You must define a method matching the variant `#{variant.name}` in your experiment `#{metric}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
+                "Undefined local method `#{varmeth}`. You must define a method matching the variant `#{variant.name}` in your experiment `#{key}`. In this case it looks like you need to define #{context.class.name}##{varmeth}(metadata={})"
             end
           end
 
@@ -181,6 +184,8 @@ module TrailGuide
 
       def run(methods: nil, **opts)
         run!(methods: methods, **opts)
+      rescue NoExperimentsError => e
+        raise e
       rescue => e
         TrailGuide.logger.error e
         false
@@ -202,19 +207,41 @@ module TrailGuide
 
       def render(prefix: nil, templates: nil, locals: {}, **opts)
         render!(prefix: prefix, templates: templates, locals: locals, **opts)
+      rescue NoExperimentsError => e
+        raise e
       rescue => e
         TrailGuide.logger.error e
         false
       end
 
       def convert!(checkpoint=nil, **opts, &block)
-        checkpoints = experiments.map { |experiment| experiment.convert!(checkpoint, **opts) }
+        raise NoExperimentsError, key if experiments.empty?
+        checkpoints = experiments.map do |experiment|
+          ckpt = checkpoint || experiment.goals.find { |g| g == key }
+          if experiment.combined?
+            experiment.combined_experiments.map do |combo|
+              combo.convert!(ckpt, **opts)
+            end
+          else
+            experiment.convert!(ckpt, **opts)
+          end
+        end.flatten
+
         return false unless checkpoints.any?
+
         if block_given?
           yield checkpoints, opts[:metadata]
         else
           checkpoints
         end
+      rescue NoExperimentsError => e
+        unless TrailGuide.configuration.ignore_orphaned_groups?
+          trace = e.backtrace.find { |t| !t.match?(Regexp.new(__FILE__)) }
+            .to_s.split(Rails.root.to_s).last
+            .split(':').first(2).join(':')
+          TrailGuide.catalog.orphaned(key, trace)
+        end
+        false
       end
 
       def convert(checkpoint=nil, **opts, &block)
@@ -225,7 +252,7 @@ module TrailGuide
       end
 
       def experiments
-        @experiments ||= TrailGuide.catalog.select(metric).map do |experiment|
+        @experiments ||= TrailGuide.catalog.select(key).map do |experiment|
           experiment.new(participant)
         end
       end

data/lib/trail_guide/metrics/checkpoint.rb

@@ -0,0 +1,24 @@
+module TrailGuide
+  module Metrics
+    # represents a checkpoint in a funnel
+    # TODO this is mostly a placeholder for now
+    class Checkpoint < Goal
+      attr_reader :experiment, :name
+
+      def dup(experiment)
+        self.class.new(experiment, name)
+      end
+
+      def initialize(experiment, name, checkpoints=[])
+        @experiment = experiment
+        @name = name.to_s.underscore.to_sym
+      end
+
+      def as_json(opts={})
+        {
+          name: name,
+        }
+      end
+    end
+  end
+end

data/lib/trail_guide/metrics/config.rb

@@ -0,0 +1,45 @@
+module TrailGuide
+  module Metrics
+    class Config < Canfig::Config
+      attr_reader :metric
+
+      INHERIT_KEYS = [ :allow_multiple_conversions, :allow_conversion, :on_convert ]
+      CALLBACK_KEYS = [ :allow_conversion, :on_convert ]
+
+      def initialize(metric, *args, **opts, &block)
+        @metric = metric
+
+        opts = INHERIT_KEYS.map do |key|
+          val = experiment.configuration[key]
+          val = val.dup if val
+          [ key, val ]
+        end.to_h.merge(opts)
+
+        super(*args, **opts, &block)
+      end
+
+      def experiment
+        metric.experiment
+      end
+
+      def callbacks
+        to_h.slice(*CALLBACK_KEYS).map { |k,v| [k, [v].flatten.compact] }.to_h
+      end
+
+      def allow_multiple_conversions?
+        !!allow_multiple_conversions
+      end
+
+      # TODO do we allow a method here? do we call it on the experiment?
+      def allow_conversion(meth=nil, &block)
+        self[:allow_conversion] ||= []
+        self[:allow_conversion] << (meth || block)
+      end
+
+      def on_convert(meth=nil, &block)
+        self[:on_convert] ||= []
+        self[:on_convert] << (meth || block)
+      end
+    end
+  end
+end

data/lib/trail_guide/metrics/funnel.rb

@@ -0,0 +1,24 @@
+module TrailGuide
+  module Metrics
+    # represents a funnel of goal checkpoints
+    # TODO this is mostly a placeholder for now
+    class Funnel < Goal
+      attr_reader :experiment, :name
+
+      def dup(experiment)
+        self.class.new(experiment, name)
+      end
+
+      def initialize(experiment, name, checkpoints=[])
+        @experiment = experiment
+        @name = name.to_s.underscore.to_sym
+      end
+
+      def as_json(opts={})
+        {
+          name: name,
+        }
+      end
+    end
+  end
+end

data/lib/trail_guide/metrics/goal.rb

@@ -0,0 +1,89 @@
+module TrailGuide
+  module Metrics
+    # represents a simple conversion goal
+    class Goal
+      attr_reader :experiment, :name
+
+      delegate :allow_multiple_conversions?, :callbacks, to: :configuration
+
+      def dup(experiment)
+        self.class.new(experiment, name, **configuration.to_h.map { |k,v| [k, v.try(:dup)] }.to_h)
+      end
+
+      def configuration
+        @configuration ||= Metrics::Config.new(self)
+      end
+
+      def configure(*args, &block)
+        configuration.configure(*args, &block)
+      end
+
+      def initialize(experiment, name, **config, &block)
+        @experiment = experiment
+        @name = name.to_s.underscore.to_sym
+        configure(**config, &block)
+      end
+
+      def ==(other)
+        if other.is_a?(self.class)
+          return name == other.name
+        elsif other.is_a?(String) || other.is_a?(Symbol)
+          other = other.to_s.underscore
+          return name == other.to_sym || to_s == other
+        elsif other.is_a?(Array)
+          return to_s == other.flatten.map { |o| o.to_s.underscore }.join('/')
+        elsif other.is_a?(Hash)
+          # TODO "flatten" it out and compare it to_s
+          return false
+        end
+      end
+
+      def ===(other)
+        return false unless other.is_a?(self.class)
+        return name == other.name && experiment == other.experiment
+      end
+
+      def allow_conversion?(trial, variant, metadata=nil)
+        return true if callbacks[:allow_conversion].empty?
+        run_callbacks(:allow_conversion, trial, variant, trial.participant, metadata)
+      end
+
+      def run_callbacks(hook, trial, *args)
+        return unless callbacks[hook]
+        if [:allow_conversion].include?(hook)
+          callbacks[hook].reduce(args.slice!(0,1)[0]) do |result, callback|
+            if callback.respond_to?(:call)
+              callback.call(trial, result, self, *args)
+            else
+              trial.send(callback, trial, result, self, *args)
+            end
+          end
+        else
+          args.unshift(self)
+          args.unshift(trial)
+          callbacks[hook].each do |callback|
+            if callback.respond_to?(:call)
+              callback.call(*args)
+            else
+              trial.send(callback, *args)
+            end
+          end
+        end
+      end
+
+      def as_json(opts={})
+        {
+          name: name,
+        }
+      end
+
+      def to_s
+        name.to_s
+      end
+
+      def storage_key
+        "#{experiment.experiment_name}:#{name}"
+      end
+    end
+  end
+end

data/lib/trail_guide/metrics.rb

@@ -0,0 +1,9 @@
+require 'trail_guide/metrics/config'
+require 'trail_guide/metrics/goal'
+require 'trail_guide/metrics/funnel'
+require 'trail_guide/metrics/checkpoint'
+
+module TrailGuide
+  module Metrics
+  end
+end

data/lib/trail_guide/participant.rb

@@ -6,7 +6,7 @@ module TrailGuide
     def initialize(context, adapter: nil)
       @context = context
       @adapter = adapter.new(context) if adapter.present?
-      cleanup_inactive_experiments! if TrailGuide.configuration.cleanup_participant_experiments
+      cleanup_inactive_experiments! if TrailGuide.configuration.cleanup_participant_experiments == true
     end
 
     def adapter
@@ -36,14 +36,15 @@ module TrailGuide
     end
 
     def variant(experiment)
-      return nil unless experiment.started?
+      return nil unless experiment.calibrating? || experiment.started?
       return nil unless adapter.key?(experiment.storage_key)
       varname = adapter[experiment.storage_key]
       variant = experiment.variants.find { |var| var == varname }
       return nil unless variant && adapter.key?(variant.storage_key)
 
       chosen_at = Time.at(adapter[variant.storage_key].to_i)
-      return variant if chosen_at >= experiment.started_at
+      started_at = experiment.started_at
+      return variant if (variant.control? && experiment.calibrating?) || (started_at && chosen_at >= started_at)
     end
 
     def participating?(experiment, include_control=true)
@@ -54,27 +55,29 @@ module TrailGuide
     end
 
     def converted?(experiment, checkpoint=nil)
-      return false unless experiment.started?
+      variant = variant(experiment)
+
+      return false unless experiment.started? || (experiment.calibrating? && variant.try(:control?))
+
       if experiment.goals.empty?
         raise InvalidGoalError, "You provided the checkpoint `#{checkpoint}` but the experiment `#{experiment.experiment_name}` does not have any goals defined." unless checkpoint.nil?
         storage_key = "#{experiment.storage_key}:converted"
         return false unless adapter.key?(storage_key)
 
         converted_at = Time.at(adapter[storage_key].to_i)
-        converted_at >= experiment.started_at
+        (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
       elsif !checkpoint.nil?
-        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." unless experiment.goals.any? { |goal| goal == checkpoint.to_s.underscore.to_sym }
-        storage_key = "#{experiment.storage_key}:#{checkpoint.to_s.underscore}"
-        return false unless adapter.key?(storage_key)
+        goal = experiment.goals.find { |g| g == checkpoint }
+        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." if goal.nil?
+        return false unless adapter.key?(goal.storage_key)
 
-        converted_at = Time.at(adapter[storage_key].to_i)
-        converted_at >= experiment.started_at
+        converted_at = Time.at(adapter[goal.storage_key].to_i)
+        (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
       else
         experiment.goals.each do |goal|
-          storage_key = "#{experiment.storage_key}:#{goal.to_s}"
-          next unless adapter.key?(storage_key)
-          converted_at = Time.at(adapter[storage_key].to_i)
-          return true if converted_at >= experiment.started_at
+          next unless adapter.key?(goal.storage_key)
+          converted_at = Time.at(adapter[goal.storage_key].to_i)
+          return true if (experiment.calibrating? && variant.try(:control?)) || converted_at >= experiment.started_at
         end
         return false
       end
@@ -86,16 +89,18 @@ module TrailGuide
     end
 
     def converted!(variant, checkpoint=nil, reset: false)
-      checkpoint ||= :converted
-      storage_key = "#{variant.experiment.storage_key}:#{checkpoint.to_s.underscore}"
+      if checkpoint.nil?
+        storage_key = "#{variant.experiment.storage_key}:converted"
+      else
+        storage_key = variant.experiment.goals.find { |g| g == checkpoint }.storage_key
+      end
 
       if reset
         adapter.delete(variant.experiment.storage_key)
         adapter.delete(variant.storage_key)
         adapter.delete(storage_key)
         variant.experiment.goals.each do |goal|
-          goal_key = "#{variant.experiment.storage_key}:#{goal.to_s}"
-          adapter.delete(goal_key)
+          adapter.delete(goal.storage_key)
         end
       else
         adapter[storage_key] = Time.now.to_i
@@ -107,17 +112,45 @@ module TrailGuide
       return true if chosen.nil?
       adapter.delete(experiment.storage_key)
       adapter.delete(chosen.storage_key)
+      adapter.delete("#{experiment.storage_key}:converted")
       experiment.goals.each do |goal|
-        adapter.delete("#{experiment.storage_key}:#{goal.to_s}")
+        adapter.delete(goal.storage_key)
       end
       return true
     end
 
     def active_experiments(include_control=true)
       return false if adapter.keys.empty?
+
+      inactive = []
+      active = adapter.keys.map { |key| key.to_s.split(":").first.to_sym }.uniq.map do |key|
+        experiment = TrailGuide.catalog.find(key)
+        next unless experiment
+
+        if !experiment.started? && !experiment.calibrating?
+          inactive << key
+          next
+        else
+          next unless !experiment.combined? && experiment.running? && participating?(experiment, include_control)
+          [ experiment.experiment_name, adapter[experiment.storage_key] ]
+        end
+      end.compact.to_h
+
+      if TrailGuide.configuration.cleanup_participant_experiments == :inline && !inactive.empty?
+        adapter.keys.select do |key|
+          inactive.include?(key.to_s.split(":").first.to_sym)
+        end.each { |key| adapter.delete(key) }
+      end
+
+      return active
+    end
+
+    def calibrating_experiments
+      return false if adapter.keys.empty?
+
       adapter.keys.map { |key| key.to_s.split(":").first.to_sym }.uniq.map do |key|
         experiment = TrailGuide.catalog.find(key)
-        next unless experiment && !experiment.combined? && experiment.running? && participating?(experiment, include_control)
+        next unless experiment && experiment.calibrating?
         [ experiment.experiment_name, adapter[experiment.storage_key] ]
       end.compact.to_h
     end
@@ -138,7 +171,7 @@ module TrailGuide
       adapter.keys.each do |key|
         experiment_name = key.to_s.split(":").first.to_sym
         experiment = TrailGuide.catalog.find(experiment_name)
-        if !experiment || !experiment.started?
+        if !experiment || (!experiment.started? && !experiment.calibrating?)
           adapter.delete(key)
         end
       end

data/lib/trail_guide/variant.rb

@@ -16,12 +16,19 @@ module TrailGuide
 
     def ==(other)
       if other.is_a?(self.class)
+        # TODO eventually remove the experiment requirement here once we start
+        # taking advantage of === below
         return name == other.name && experiment == other.experiment
       elsif other.is_a?(String) || other.is_a?(Symbol)
         return name == other.to_s.underscore.to_sym
       end
     end
 
+    def ===(other)
+      return false unless other.is_a?(self.class)
+      return name == other.name && experiment == other.experiment
+    end
+
     # TODO maybe track the control on the experiment itself, rather than as a
     # flag on the variants like this?
 
@@ -66,11 +73,12 @@ module TrailGuide
         raise InvalidGoalError, "You provided the checkpoint `#{checkpoint}` but the experiment `#{experiment.experiment_name}` does not have any goals defined." unless checkpoint.nil?
         (TrailGuide.redis.hget(storage_key, 'converted') || 0).to_i
       elsif !checkpoint.nil?
-        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." unless experiment.goals.any? { |goal| goal == checkpoint.to_s.underscore.to_sym }
-        (TrailGuide.redis.hget(storage_key, checkpoint.to_s.underscore) || 0).to_i
+        goal = experiment.goals.find { |g| g == checkpoint }
+        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for experiment `#{experiment.experiment_name}`." if goal.nil?
+        (TrailGuide.redis.hget(storage_key, goal.to_s) || 0).to_i
       else
-        experiment.goals.sum do |checkpoint|
-          (TrailGuide.redis.hget(storage_key, checkpoint.to_s.underscore) || 0).to_i
+        experiment.goals.sum do |goal|
+          (TrailGuide.redis.hget(storage_key, goal.to_s) || 0).to_i
         end
       end
     end
@@ -79,22 +87,36 @@ module TrailGuide
       participants - converted
     end
 
+    def measure(goal=nil, against=nil)
+      superset = against ? converted(against) : participants
+      converts = converted(goal)
+      return 0 if superset.zero? || converts.zero?
+      converts.to_f / superset.to_f
+    end
+
     def increment_participation!
       TrailGuide.redis.hincrby(storage_key, 'participants', 1)
     end
 
     def increment_conversion!(checkpoint=nil)
-      checkpoint ||= :converted
-      TrailGuide.redis.hincrby(storage_key, checkpoint.to_s.underscore, 1)
+      if checkpoint.nil?
+        checkpoint = 'converted'
+      else
+        checkpoint = experiment.goals.find { |g| g == checkpoint }.to_s
+      end
+      TrailGuide.redis.hincrby(storage_key, checkpoint, 1)
     end
 
+    # export the variant state (not config) as json
     def as_json(opts={})
-      {
-        name: name,
-        control: control?,
-        weight: weight,
-        metadata: metadata.as_json,
-      }
+      if experiment.goals.empty?
+        conversions = converted
+      else
+        conversions = experiment.goals.map { |g| [g.name, converted(g)] }.to_h
+      end
+
+      { name => { participants: participants,
+                  converted: conversions } }
     end
 
     def to_s

data/lib/trail_guide/version.rb

@@ -1,8 +1,8 @@
 module TrailGuide
   module Version
     MAJOR = 0
-    MINOR = 1
-    PATCH = 31
+    MINOR = 2
+    PATCH = 0
     VERSION = "#{MAJOR}.#{MINOR}.#{PATCH}"
 
     class << self

data/lib/trailguide.rb

@@ -5,6 +5,8 @@ require "trail_guide/config"
 require "trail_guide/errors"
 require "trail_guide/adapters"
 require "trail_guide/algorithms"
+require "trail_guide/metrics"
+require "trail_guide/calculators"
 require "trail_guide/participant"
 require "trail_guide/variant"
 require "trail_guide/experiment"
@@ -18,6 +20,8 @@ module TrailGuide
   include Canfig::Module
   @@configuration = TrailGuide::Config.new
 
+  SCHEDULE_DATE_FORMAT = "%m/%d/%Y %I:%M %p %Z"
+
   class << self
     delegate :logger, :redis, to: :configuration
   end