trailguide 0.1.31 → 0.2.0

data/lib/trail_guide/experiments/base.rb

@@ -5,12 +5,16 @@ module TrailGuide
   module Experiments
     class Base
       class << self
-        delegate :metric, :algorithm, :control, :goals, :callbacks, :combined,
+        delegate :groups, :algorithm, :control, :goals, :callbacks, :combined,
           :combined?, :allow_multiple_conversions?, :allow_multiple_goals?,
           :track_winner_conversions?, :start_manually?, :reset_manually?,
-          to: :configuration
+          :enable_calibration?, to: :configuration
         alias_method :funnels, :goals
 
+        def register!
+          TrailGuide.catalog.register(self)
+        end
+
         def configuration
           @configuration ||= Experiments::Config.new(self)
         end
@@ -31,6 +35,14 @@ module TrailGuide
           end
         end
 
+        def is_combined?
+          false
+        end
+
+        def combined_experiments
+          combined.map { |combo| TrailGuide.catalog.find(combo) }
+        end
+
         def run_callbacks(hook, *args)
           return unless callbacks[hook]
           return args[0] if hook == :rollout_winner
@@ -52,6 +64,15 @@ module TrailGuide
           started
         end
 
+        def schedule!(start_at, stop_at=nil, context=nil)
+          return false if started?
+          save! unless persisted?
+          scheduled = TrailGuide.redis.hset(storage_key, 'started_at', start_at.to_i)
+          TrailGuide.redis.hset(storage_key, 'stopped_at', stop_at.to_i) if stop_at
+          run_callbacks(:on_schedule, start_at, stop_at, context)
+          scheduled
+        end
+
         def pause!(context=nil)
           return false unless running? && configuration.can_resume?
           paused = TrailGuide.redis.hset(storage_key, 'paused_at', Time.now.to_i)
@@ -89,21 +110,33 @@ module TrailGuide
         end
 
         def started?
-          started_at && started_at <= Time.now
+          time = started_at
+          time && time <= Time.now
+        end
+
+        def scheduled?
+          time = started_at
+          time && time > Time.now
         end
 
         def paused?
-          paused_at && paused_at <= Time.now
+          time = paused_at
+          time && time <= Time.now
         end
 
         def stopped?
-          stopped_at && stopped_at <= Time.now
+          time = stopped_at
+          time && time <= Time.now
         end
 
         def running?
           started? && !paused? && !stopped?
         end
 
+        def calibrating?
+          enable_calibration? && start_manually? && !started?
+        end
+
         def declare_winner!(variant, context=nil)
           variant = variants.find { |var| var == variant } unless variant.is_a?(Variant)
           run_callbacks(:on_winner, variant, context)
@@ -120,7 +153,11 @@ module TrailGuide
         end
 
         def winner?
-          TrailGuide.redis.hexists(storage_key, 'winner')
+          if combined?
+            combined.all? { |combo| TrailGuide.catalog.find(combo).winner? }
+          else
+            TrailGuide.redis.hexists(storage_key, 'winner')
+          end
         end
 
         def persisted?
@@ -165,23 +202,14 @@ module TrailGuide
           return false
         end
 
+        # export the experiment state (not config) as json
         def as_json(opts={})
           { experiment_name => {
-            configuration: {
-              metric: metric,
-              algorithm: algorithm.name,
-              variants: variants.as_json,
-              goals: goals.as_json,
-              start_manually: start_manually?,
-              reset_manually: reset_manually?,
-              allow_multiple_conversions: allow_multiple_conversions?,
-              allow_multiple_goals: allow_multiple_goals?
-            },
-            statistics: {
-              # TODO expand on this for variants/goals
-              participants: variants.sum(&:participants),
-              converted: variants.sum(&:converted)
-            }
+            started_at: started_at,
+            paused_at: paused_at,
+            stopped_at: stopped_at,
+            winner: winner.try(:name),
+            variants: variants.map(&:as_json).reduce({}) { |r,v| r.merge!(v) },
           } }
         end
 
@@ -192,9 +220,9 @@ module TrailGuide
 
       attr_reader :participant
       delegate :configuration, :experiment_name, :variants, :control, :goals,
-        :storage_key, :running?, :started?, :started_at, :start!,
-        :start_manually?, :reset_manually?, :winner?, :allow_multiple_conversions?,
-        :allow_multiple_goals?, :track_winner_conversions?, :callbacks, to: :class
+        :storage_key, :combined?, :start_manually?, :reset_manually?,
+        :allow_multiple_conversions?, :allow_multiple_goals?, :is_combined?,
+        :enable_calibration?, :track_winner_conversions?, :callbacks, to: :class
 
       def initialize(participant)
         @participant = TrailGuide::Experiments::Participant.new(self, participant)
@@ -204,15 +232,15 @@ module TrailGuide
         @algorithm ||= self.class.algorithm.new(self)
       end
 
-      def winner
-        run_callbacks(:rollout_winner, self.class.winner)
+      def winning_variant
+        run_callbacks(:rollout_winner, self.class.winner, participant)
       end
 
       def choose!(override: nil, metadata: nil, **opts)
         return control if TrailGuide.configuration.disabled
 
         variant = choose_variant!(override: override, metadata: metadata, **opts)
-        run_callbacks(:on_use, variant, metadata)
+        run_callbacks(:on_use, variant, participant, metadata)
         variant
       rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
         run_callbacks(:on_redis_failover, e)
@@ -225,7 +253,7 @@ module TrailGuide
 
         if override.present?
           variant = variants.find { |var| var == override } || control
-          if running?
+          if running? && !is_combined?
             variant.increment_participation! if configuration.track_override
             participant.participating!(variant) if configuration.store_override
           end
@@ -233,14 +261,36 @@ module TrailGuide
         end
 
         if winner?
-          variant = winner
-          variant.increment_participation! if track_winner_conversions?
+          variant = winning_variant
+          if track_winner_conversions? && running?
+            variant.increment_participation! unless participant.variant == variant
+            participant.exit! if participant.participating? && participant.variant != variant
+            participant.participating!(variant)
+          end
           return variant
         end
 
-        return control if excluded
-        return control if !started? && configuration.start_manually
-        start! unless started?
+        return control if excluded || stopped?
+
+        if !started? && start_manually?
+          if enable_calibration?
+            unless participant.variant == control
+              control.increment_participation!
+              parent.control.increment_participation! if is_combined?
+            end
+
+            if participant.participating? && participant.variant != control
+              participant.exit!
+              parent.participant.exit! if is_combined?
+            end
+
+            participant.participating!(control)
+            parent.participant.participating!(parent.control) if is_combined?
+          end
+          return control
+        end
+
+        start! unless started? || scheduled?
         return control unless running?
 
         if participant.participating?
@@ -249,11 +299,13 @@ module TrailGuide
           return variant
         end
 
-        return control unless TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
+        return control unless is_combined? || TrailGuide.configuration.allow_multiple_experiments == true || !participant.participating_in_active_experiments?(TrailGuide.configuration.allow_multiple_experiments == false)
         return control unless allow_participation?(metadata)
 
         variant = algorithm_choose!(metadata: metadata)
-        variant_chosen!(variant, metadata: metadata)
+        variant.increment_participation!
+        participant.participating!(variant)
+        run_callbacks(:on_choose, variant, participant, metadata)
         variant
       end
 
@@ -261,30 +313,48 @@ module TrailGuide
         algorithm.choose!(metadata: metadata)
       end
 
-      def variant_chosen!(variant, metadata: nil)
-        variant.increment_participation!
-        participant.participating!(variant)
-        run_callbacks(:on_choose, variant, metadata)
-      end
-
       def convert!(checkpoint=nil, metadata: nil)
-        return false if !running? || (winner? && !track_winner_conversions?)
-        return false unless participant.participating?
-        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.present? || goals.empty?
-        raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." unless checkpoint.nil? || goals.any? { |goal| goal == checkpoint.to_s.underscore.to_sym }
+        if !started?
+          return false unless enable_calibration?
+          variant = participant.variant
+          return false unless variant.present? && variant == control
+        else
+          return false unless running?
+          variant = participant.variant
+          return false unless variant.present?
+
+          if winner?
+            return false unless track_winner_conversions? && variant == winner
+          end
+        end
+
+        if checkpoint.nil?
+          raise InvalidGoalError, "You must provide a valid goal checkpoint for `#{experiment_name}`." unless goals.empty?
+        else
+          goal = goals.find { |g| g == checkpoint }
+          raise InvalidGoalError, "Invalid goal checkpoint `#{checkpoint}` for `#{experiment_name}`." if goal.nil?
+          checkpoint = goal
+        end
+
         # TODO eventually allow progressing through funnel checkpoints towards goals
         if participant.converted?(checkpoint)
-          return false unless allow_multiple_conversions?
+          return false unless (checkpoint.nil? && allow_multiple_conversions?) || (checkpoint.present? && checkpoint.allow_multiple_conversions?)
         elsif participant.converted?
           return false unless allow_multiple_goals?
         end
-        return false unless allow_conversion?(checkpoint, metadata)
+        return false unless allow_conversion?(variant, checkpoint, metadata)
 
-        variant = participant.variant
+        # TODO only reset if !reset_manually? AND they've converted all goals if
+        # allow_multiple_goals? is set
+        # TODO what should happen when allow_multiple_conversions? and !reset_manually?
         # TODO eventually only reset if we're at the final goal in a funnel
         participant.converted!(variant, checkpoint, reset: !reset_manually?)
         variant.increment_conversion!(checkpoint)
-        run_callbacks(:on_convert, variant, checkpoint, metadata)
+        if checkpoint.nil?
+          run_callbacks(:on_convert, checkpoint, variant, participant, metadata)
+        else
+          checkpoint.run_callbacks(:on_convert, self, variant, participant, metadata)
+        end
         variant
       rescue Errno::ECONNREFUSED, Redis::BaseError, SocketError => e
         run_callbacks(:on_redis_failover, e)
@@ -293,12 +363,16 @@ module TrailGuide
 
       def allow_participation?(metadata=nil)
         return true if callbacks[:allow_participation].empty?
-        run_callbacks(:allow_participation, metadata)
+        run_callbacks(:allow_participation, participant, metadata)
       end
 
-      def allow_conversion?(checkpoint=nil, metadata=nil)
-        return true if callbacks[:allow_conversion].empty?
-        run_callbacks(:allow_conversion, checkpoint, metadata)
+      def allow_conversion?(variant, checkpoint=nil, metadata=nil)
+        if checkpoint.nil?
+          return true if callbacks[:allow_conversion].empty?
+          run_callbacks(:allow_conversion, checkpoint, variant, participant, metadata)
+        else
+          checkpoint.allow_conversion?(self, variant, metadata)
+        end
       end
 
       def run_callbacks(hook, *args)
@@ -322,6 +396,68 @@ module TrailGuide
           end
         end
       end
+
+      def combined_experiments
+        @combined_experiments ||= self.class.combined_experiments.map do |combo|
+          combo.new(participant.participant)
+        end
+      end
+
+      ### MEMOIZATOIN ###
+      # This is a lot of seemingly unnecessary duplication, but it really helps
+      # to cut down on the number of redis requests while still being
+      # thread-safe by memoizing these methods/values here at the instance level
+
+      def start!
+        @started_at = nil
+        self.class.start!
+      end
+
+      def started_at
+        @started_at ||= self.class.started_at
+      end
+
+      def paused_at
+        @paused_at ||= self.class.paused_at
+      end
+
+      def stopped_at
+        @stopped_at ||= self.class.stopped_at
+      end
+
+      def winner
+        @winner ||= self.class.winner
+      end
+
+      def scheduled?
+        started_at && started_at > Time.now
+      end
+
+      def started?
+        started_at && started_at <= Time.now
+      end
+
+      def paused?
+        paused_at && paused_at <= Time.now
+      end
+
+      def stopped?
+        stopped_at && stopped_at <= Time.now
+      end
+
+      def running?
+        started? && !paused? && !stopped?
+      end
+
+      def calibrating?
+        enable_calibration? && start_manually? && !started?
+      end
+
+      def winner?
+        return @has_winner unless @has_winner.nil?
+        @has_winner = self.class.winner?
+      end
+
     end
   end
 end

data/lib/trail_guide/experiments/config.rb

@@ -2,17 +2,18 @@ module TrailGuide
   module Experiments
     class Config < Canfig::Config
       DEFAULT_KEYS = [
-        :name, :summary, :preview_url, :algorithm, :metric, :variants, :goals,
-        :start_manually, :reset_manually, :store_override, :track_override,
-        :combined, :allow_multiple_conversions, :allow_multiple_goals,
-        :track_winner_conversions, :skip_request_filter, :target_sample_size,
-        :can_resume
+        :name, :summary, :preview_url, :algorithm, :groups, :variants, :goals,
+        :start_manually, :reset_manually, :store_participation, :store_override,
+        :track_override, :combined, :allow_multiple_conversions,
+        :allow_multiple_goals, :track_winner_conversions, :skip_request_filter,
+        :target_sample_size, :can_resume, :enable_calibration
       ].freeze
 
       CALLBACK_KEYS = [
-        :on_start, :on_stop, :on_pause, :on_resume, :on_winner, :on_reset,
-        :on_delete, :on_choose, :on_use, :on_convert, :on_redis_failover,
-        :allow_participation, :allow_conversion, :rollout_winner
+        :on_start, :on_schedule, :on_stop, :on_pause, :on_resume, :on_winner,
+        :on_reset, :on_delete, :on_choose, :on_use, :on_convert,
+        :on_redis_failover, :allow_participation, :allow_conversion,
+        :rollout_winner
       ].freeze
 
       def default_config
@@ -57,6 +58,10 @@ module TrailGuide
         !!reset_manually
       end
 
+      def store_participation?
+        !!store_participation
+      end
+
       def allow_multiple_conversions?
         !!allow_multiple_conversions
       end
@@ -77,12 +82,37 @@ module TrailGuide
         !!can_resume
       end
 
+      def enable_calibration?
+        !!enable_calibration
+      end
+
       def name
         @name ||= (self[:name] || experiment.name).try(:to_s).try(:underscore).try(:to_sym)
       end
 
-      def metric
-        @metric ||= (self[:metric] || name).try(:to_s).try(:underscore).try(:to_sym)
+      def groups(*grps)
+        self[:groups] ||= []
+        unless grps.empty?
+          self[:groups] = self[:groups].concat([grps].flatten.map { |g| g.to_s.underscore.to_sym })
+        end
+        self[:groups]
+      end
+
+      def groups=(*grps)
+        self[:groups] = [grps].flatten.map { |g| g.to_s.underscore.to_sym }
+      end
+
+      def group(grp=nil)
+        unless grp.nil?
+          groups << grp.to_s.underscore.to_sym
+          return groups.last
+        end
+        groups.first
+      end
+
+      def group=(grp)
+        groups.unshift(grp.to_s.underscore.to_sym)
+        groups.first
       end
 
       def algorithm
@@ -117,16 +147,50 @@ module TrailGuide
         variant
       end
 
-      def goal(name)
-        goals << name.to_s.underscore.to_sym
+      def goal(name, **config, &block)
+        goals << Metrics::Goal.new(experiment, name, **config, &block)
       end
       alias_method :funnel, :goal
 
-      def goals
+      def goal=(name)
+        goals << Metrics::Goal.new(experiment, name)
+      end
+      alias_method :funnel=, :goal=
+
+      def goals=(*names)
+        self[:goals] = [names].flatten.map { |g| Metrics::Goal.new(experiment, g) }
+      end
+      alias_method :funnels=, :goals=
+
+      def goals(*names, **config, &block)
+        self[:goals] ||= []
+        unless names.empty?
+          self[:goals] = self[:goals].concat([names].flatten.map { |g| Metrics::Goal.new(experiment, g, **config, &block) })
+        end
         self[:goals]
       end
       alias_method :funnels, :goals
 
+      def metric(name, **config, &block)
+        group(name)
+        goal(name, **config, &block)
+      end
+
+      def metric=(name)
+        self.group = name
+        self.goal = name
+      end
+
+      def metrics(*names, **config, &block)
+        groups(*names)
+        goals(*names, **config, &block)
+      end
+
+      def metrics=(*names)
+        self.groups = *names
+        self.goals = *names
+      end
+
       def combined?
         !combined.empty?
       end
@@ -159,6 +223,11 @@ module TrailGuide
         self[:on_start] << (meth || block)
       end
 
+      def on_schedule(meth=nil, &block)
+        self[:on_schedule] ||= []
+        self[:on_schedule] << (meth || block)
+      end
+
       def on_stop(meth=nil, &block)
         self[:on_stop] ||= []
         self[:on_stop] << (meth || block)

data/lib/trail_guide/experiments/participant.rb

@@ -12,14 +12,34 @@ module TrailGuide
         @participating ||= variant.present?
       end
 
+      def participating!(variant)
+        @participating = true
+        @variant = variant
+        participant.participating!(variant) if experiment.configuration.store_participation?
+      end
+
       def converted?(checkpoint=nil)
-        @converted ||= participant.converted?(experiment, checkpoint)
+        @converted ||= {}
+        @converted[checkpoint || :converted] ||= participant.converted?(experiment, checkpoint)
+      end
+
+      def converted!(variant, checkpoint, reset: false)
+        @converted ||= {}
+        @converted[checkpoint || :converted] ||= true
+        participant.converted!(variant, checkpoint, reset: reset)
       end
 
       def variant
         @variant ||= participant.variant(experiment)
       end
 
+      def exit!
+        @participating = nil
+        @converted = nil
+        @variant = nil
+        participant.exit!(experiment)
+      end
+
       def method_missing(meth, *args, &block)
         return participant.send(meth, *args, &block) if participant.respond_to?(meth, true)
         super