activeexperiment 0.1.0.alpha

data/lib/active_experiment/caching.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "active_support/cache"
+
+module ActiveExperiment
+  # == Caching
+  #
+  # Active Experiment can use caching for variant assignments. Caching is a
+  # complex topic, and unlike a lot of caching strategies where a key can
+  # expire and/or be cleaned up automatically, Active Experiment requires a
+  # cache store that will hold a given cache key for the lifetime of the
+  # experiment.
+  #
+  # Since an experiment cache has to live for the lifetime of the experiment,
+  # there are some special considerations about the size of the cache and how
+  # we might clean it up after an experiment is removed, and also if/when to
+  # use caching at all.
+  #
+  # == When to Use Caching
+  #
+  # In simple experiments caching may not be required, but as experiments get
+  # more complex, caching starts to become a more important aspect to consider.
+  # Because of this, caching can be configured on a per experiment basis.
+  #
+  # When should you consider caching? Exclusions and segmenting rules can often
+  # benefit from caching, and so it should be considered whenever adding
+  # segment rules to an experiment.
+  #
+  # For example, here's an experiment that highlights why caching can be an
+  # important consideration when adding a segment rule:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     segment :older_accounts, into: :red
+  #
+  #     private
+  #
+  #     def older_accounts
+  #       context.created_at < 1.week.ago
+  #     end
+  #   end
+  #
+  # If caching isn't used for this experiment, a new account might be assigned
+  # the blue variant initially, and within a week the variant would switch to
+  # red because they shift into the "older_accounts" segment.
+  #
+  # In some scenarios it might be desirable to allow contexts to move between
+  # the segments, but in most cases it's not.
+  #
+  # == Cache Considerations
+  #
+  # The cache store used should be a long lived cache, such as Redis, or even a
+  # database. The cache store should also be able to handle the number of keys
+  # that will be stored in it.
+  #
+  # For example, if you have 100 users and 100 posts, and define an experiment
+  # that runs on all users viewing all posts, you'll have a cache potential of
+  # ~1,000,000 entries for that experiment alone.
+  #
+  # Here's an example of what that means, and how you can consider the cache
+  # size:
+  #
+  #   User.count # => 100
+  #   Post.count # => 100
+  #   MyExperiment.run(user: user, post: post) # => cache potential: ~1,000,000
+  #
+  # Now, will that potential ever be hit? It's hard to say, and the answer is
+  # dependent on where the experiment is being run, and other considerations
+  # like those.
+  #
+  # If the same experiment is run only on posts (or users), the cache potential
+  # would be limited to 100.
+  #
+  # == Custom Cache Stores
+  #
+  # Custom cache stores can be created and registered, as long as they adhere
+  # to the standard interface in +ActiveSupport::Cache::Store+ they can be
+  # used -- provided it can handle the long lived caching nature required by
+  # Active Experiment.
+  #
+  # == Configuring Caching
+  #
+  # Caching can be configured globally, and overridden on a per experiment
+  # basis. By default the cache store used is the standard +:null_store+ as its
+  # defined in +ActiveSupport::Cache::NullStore+. This is a no-op cache store
+  # that doesn't actually cache anything but provides a consistent interface.
+  #
+  # Active Experiment ships with a functional cache store based on using the
+  # Redis hash data type (https://redis.io/docs/data-types/hashes/). This cache
+  # store expects a redis instance or pool that hasn't been configured to auto
+  # expire keys.
+  #
+  # To configure the cache store globally:
+  #
+  #   ActiveExperiment::Base.cache_store = :redis_hash
+  #
+  # To configure the cache store on a per experiment basis:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     use_cache_store :redis_hash
+  #   end
+  module Caching
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :cache_store, instance_writer: false, instance_predicate: false
+
+      self.default_cache_store = :null_store
+    end
+
+    module ClassMethods
+      def inherited(subclass)
+        super
+        subclass.default_cache_store = @default_cache_store
+      end
+
+      def default_cache_store=(name_or_cache_store)
+        use_cache_store(name_or_cache_store)
+        @default_cache_store = name_or_cache_store
+      end
+
+      def clear_cache(cache_key_prefix = nil)
+        cache_store.delete_matched(cache_key_prefix || experiment_name)
+      end
+
+      private
+        def use_cache_store(name_or_cache_store, *args, **kws)
+          case name_or_cache_store
+          when Symbol, String
+            self.cache_store = ActiveExperiment::Cache.lookup(name_or_cache_store, *args, **kws)
+          else
+            self.cache_store = name_or_cache_store
+          end
+        end
+    end
+
+    # The cache key prefix.
+    #
+    # This is used to namespace cache keys, and can be used to find all cache
+    # keys for a given experiment.
+    def cache_key_prefix
+      name
+    end
+
+    # The cache key for a given experiment and experiment context.
+    #
+    # The cache key includes the experiment name and hexdigest generated by the
+    # experiment context.
+    def cache_key
+      [cache_key_prefix, run_key.slice(0, 32)].join(":")
+    end
+
+    # Store the variant assignment in the cache.
+    #
+    # Raises an +ExecutionError+ if no variant has been assigned.
+    def cache_variant!
+      raise ExecutionError, "No variant assigned" unless variant.present?
+
+      cache_store.write(self, variant)
+    end
+
+    private
+      def cached_variant(variant, &block)
+        cache_store.fetch(self, skip_nil: true) { variant || block&.call }
+      end
+  end
+end

data/lib/active_experiment/callbacks.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "active_support/callbacks"
+
+module ActiveExperiment
+  # == Callbacks
+  #
+  # Active Experiment provides several callbacks to hook into the lifecycle of
+  # running an experiment. Using callbacks in Active Experiment is the same as
+  # using other callbacks within Rails.
+  #
+  # The callbacks are generally separated into two concepts: run and variant.
+  # Run callbacks are invoked whenever an experiment is run, and variant
+  # callbacks are only invoked when that variant is assigned or resolved.
+  #
+  # The following run callback methods are available:
+  #
+  # * +before_run+
+  # * +after_run+
+  # * +around_run+
+  #
+  # The variant may not be known when each run callback is invoked, so it's not
+  # advised to rely on a variant within run callbacks. The variant callbacks
+  # are useful for that however, since they're only invoked for a given
+  # variant. The variant name must be provided to the variant callback methods.
+  #
+  # The following variant callback methods are available:
+  #
+  # * +before_variant+
+  # * +after_variant+
+  # * +around_variant+
+  #
+  # An example of an experiment that uses run and variant callbacks:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     after_run :after_run_callback_method, if: -> { true }
+  #     before_run { puts "before #{name}" }
+  #     around_run do |_experiment, block|
+  #       puts "around #{name} [#{block.call}]"
+  #     end
+  #
+  #     after_variant(:red) { puts "after:red #{name}" }
+  #     before_variant(:red) { puts "before:red #{name}" }
+  #     around_variant(:blue) do |_experiment, block|
+  #       puts "around:blue #{name} [#{block.call}]"
+  #     end
+  #
+  #     private
+  #
+  #     def after_run_callback_method
+  #       puts "after #{name}"
+  #     end
+  #   end
+  module Callbacks
+    extend ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    included do
+      define_callbacks :run, skip_after_callbacks_if_terminated: true
+      private :__callbacks, :__callbacks?, :run_callbacks, :_run_callbacks, :_run_run_callbacks
+    end
+
+    # These methods will be included into any Active Experiment object, adding
+    # the run and variant callback methods, and tooling to build callbacks with
+    # a target, which is used by segment rules and variant steps.
+    module ClassMethods
+      private
+        def before_run(*filters, &block)
+          set_callback(:run, :before, *filters, &block)
+        end
+
+        def after_run(*filters, &block)
+          set_callback(:run, :after, *filters, &block)
+        end
+
+        def around_run(*filters, &block)
+          set_callback(:run, :around, *filters, &block)
+        end
+
+        def before_variant(variant, *filters, &block)
+          set_variant_callback(variant, :before, *filters, &block)
+        end
+
+        def after_variant(variant, *filters, &block)
+          set_variant_callback(variant, :after, *filters, &block)
+        end
+
+        def around_variant(variant, *filters, &block)
+          set_variant_callback(variant, :around, *filters, &block)
+        end
+
+        def set_callback_with_target(chain, *filters, default: nil, **options)
+          filters = filters.compact
+
+          if filters.empty? && !default.nil?
+            filters = [default] if options[:if].present? || options[:unless].present?
+          end
+
+          filters = filters.map do |filter|
+            result_lambda = ActiveSupport::Callbacks::CallTemplate.build(filter, self).make_lambda
+            ->(target) { yield(target, result_lambda) }
+          end
+
+          set_callback(chain, *filters, **options)
+        end
+    end
+  end
+end

data/lib/active_experiment/capturable.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Capturable Mixin
+  #
+  # This module adds the capability to capture and render the results of an
+  # experiment in the order that would make sense when rendering in a view.
+  #
+  # Add it to experiments that should capture and render their results.
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     include ActiveExperiment::Capturable
+  #   end
+  #
+  # The order of experiment execution is to call the run block before resolving
+  # the variant, and subsequently calling the appropriate variant block. This
+  # order allows the run block to set details that can be used in resolving the
+  # variant and/or to set the variant directly -- or to skip the experiment
+  # altogether even.
+  #
+  # The order of how code is implemented in the run block shouldn't matter to
+  # the experiment (generally speaking) and the two following examples do the
+  # same thing:
+  #
+  #   MyExperiment.run do |experiment|
+  #     experiment.skip if current_user.admin?
+  #     experiment.on(:red) { "red override" }
+  #   end
+  #
+  #   MyExperiment.run do |experiment|
+  #     experiment.on(:red) { "red override" }
+  #     experiment.skip if current_user.admin?
+  #   end
+  #
+  # This is desirable most of the time, for important performance reasons, but
+  # can also be undesirable when running an experiment in a view and wanting to
+  # capture the markup in the expected order.
+  #
+  # In the following example the container div is shared between the variants,
+  # and duplicating it (potentially several times) in each variant block would
+  # be undesirable:
+  #
+  #   <%== MyExperiment.set(capture: self).run do |experiment| %>
+  #     <div class="container">
+  #       <%= experiment.on(:red) do %>
+  #         <button class="red-pill">Red</button>
+  #       <% end %>
+  #       <%= experiment.on(:blue) do %>
+  #         <button class="blue-pill">Blue</button>
+  #       <% end %>
+  #     </div>
+  #   <% end %>
+  #
+  # There are a couple important things to note about the above example to
+  # ensure capturing works as expected:
+  #
+  # 1. The `ActiveExperiment::Capturable` module has been included in the
+  #    experiment class.
+  #
+  # 2. The use of +==+ in the ERB tag is important because Active Experiment
+  #    doesn't try to determine if the experiment results are safe to render,
+  #    and it's up to the caller to make them html safe again.
+  #
+  # 3. The +capture+ option that's passed to the +set+ method tells Active
+  #    Experiment to use the view context's +capture+ logic to build the output
+  #    in the expected order.
+  #
+  # 4. Each variant block should use +=+ on the ERB tag to ensure the variant
+  #    content ends up where it should be in the output.
+  #
+  # In HAML, the above example would look like:
+  #
+  #   != MyExperiment.set(capture: self).run do |experiment|
+  #     %div.container
+  #       = experiment.on(:red) do
+  #         %button.red-pill Red
+  #       = experiment.on(:blue) do
+  #         %button.blue-pill Blue
+  module Capturable
+    extend ActiveSupport::Concern
+
+    def on(*variant_names, &block)
+      super
+
+      "{{#{variant_names.join("}}{{")}}}"
+    end
+
+    def run(&block)
+      super
+
+      if capturable?
+        @results = @capture.to_s.gsub(/{{([\w]+)}}/) { $1 == variant.to_s ? @results : "" }
+      else
+        @results
+      end
+    end
+
+    private
+      def resolve_results
+        @results = capture { super }
+      end
+
+      def call_run_block(&block)
+        @capture = capture { super }
+      end
+
+      def capture(&block)
+        return yield unless capturable?
+
+        @options[:capture].capture(&block)
+      end
+
+      def capturable?
+        !!@options[:capture]&.respond_to?(:capture)
+      end
+  end
+end

data/lib/active_experiment/configured_experiment.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Configured Experiment
+  #
+  # A wrapper around an experiment that allows setting options on, and then
+  # doing more with the experiment. It includes tooling for running, or caching
+  # a variant for a collection of contexts.
+  #
+  # When calling +MyExperiment.set+, a configured experiment will be returned.
+  # Additional methods can then be called on the configured experiment instance
+  # to run the experiment or cache variants.
+  #
+  # For example, a configured experiment can be used to configure, instantiate
+  # and run an experiment:
+  #
+  #   MyExperiment.set(variant: :blue).run(id: 1) do |experiment|
+  #     experiment.on(:blue) { "blue override" }
+  #   end
+  #
+  # Or it can be used to cache the variant assignment for a collection of
+  # contexts.
+  #
+  #   MyExperiment.set(variant: red).cache_each(User.find_each)
+  #
+  # This class is also provided to be used when adding additional tooling for
+  # specific project needs. Here's an example of reopening this class to add a
+  # project specific +cleanup_cache+ method:
+  #
+  #   class ActiveExperiment::ConfiguredExperiment
+  #     def cleanup_cache
+  #       store = experiment.cache_store
+  #       store.delete_matched("#{experiment.cache_key_prefix}/*")
+  #     end
+  #   end
+  #
+  #  Which could then be used by calling one of the following:
+  #
+  #   MyExperiment.set.cleanup_cache
+  #   ConfiguredExperiment.new(MyExperiment).cleanup_cache
+  class ConfiguredExperiment
+    def initialize(experiment_class, **options) # :nodoc:
+      @experiment_class = experiment_class
+      @options = options
+    end
+
+    # Runs the experiment with the configured options, context, and the given
+    # block. The block will be called with the experiment instance.
+    #
+    # This is a convenience method for instantiating and running an experiment:
+    #
+    #   MyExperiment.set(variant: :blue).run(id: 1) { }
+    def run(context = {}, &block)
+      experiment(context).run(&block)
+    end
+
+    # When provided an enumerable, an experiment will be instantiated for each
+    # item and the variant assignment will be cached. This method can be used
+    # to pre-cache the variant assignment for a collection of contexts.
+    #
+    #   MyExperiment.set(variant: red).cache_each(User.find_each)
+    def cache_each(enumerable_contexts)
+      enumerable_contexts.each do |context|
+        experiment(context).cache_variant!
+      end
+    end
+
+    # Returns the experiment instance with the configured options and provided
+    # context.
+    def experiment(context = {})
+      @experiment_class.new(context).set(**@options)
+    end
+  end
+end

data/lib/active_experiment/core.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Core Module
+  #
+  # Provides general behavior that will be included into every Active
+  # Experiment object that inherits from ActiveExperiment::Base.
+  module Core
+    extend ActiveSupport::Concern
+
+    # The experiment context.
+    #
+    # Experiment contexts should be consistent and unique values that are used
+    # to assign the same variant over many runs. Examples can range from an
+    # Active Record object to the weekday name. If a given variant is assigned
+    # on Tuesdays, it will always be assigned on Tuesdays, or if a variant is
+    # assigned for a given Active Record object, it will always be assigned for
+    # that record.
+    #
+    # Context is used to generate the cache key, so including something that
+    # would change the cache key on every run is not recommended.
+    attr_reader :context
+
+    # The experiment name.
+    #
+    # This is an underscored version of the experiment class name. If within a
+    # namespace, the namespace will be included in the name, separated by a
+    # slash (e.g. "my_namespace/my_experiment").
+    attr_reader :name
+
+    # The experiment run identifier.
+    #
+    # A unique UUID, per experiment instantiation.
+    attr_reader :run_id
+
+    # The experiment run key.
+    #
+    # This is a hexdigest that's generated from the experiment context. The run
+    # key is used as the cache key and can be used by the rollout to determine
+    # variant assignment.
+    attr_reader :run_key
+
+    # The variant that's been assigned or resolved for this run.
+    #
+    # This can be manually provided before the experiment is run, or can be
+    # resolved by segment rules or asking the rollout.
+    attr_reader :variant
+
+    # Experiment options.
+    #
+    # Generally not used within the core library, this is provided to expose
+    # additional data when running experiments. Use the +set+ method to set a
+    # variant, or other options.
+    attr_reader :options
+
+    # These methods will be included into any Active Experiment object and
+    # provide variant registration methods.
+    module ClassMethods
+      # The experiment name.
+      #
+      # An underscored version of the experiment class name. If within a
+      # namespace, the namespace will be included in the name, separated by a
+      # slash (e.g. "my_namespace/my_experiment").
+      def experiment_name
+        name.underscore
+      end
+
+      private
+        def control(...)
+          variant(:control, ...)
+        end
+
+        def variant(name, ...)
+          register_variant_callback(name, ...)
+        end
+    end
+
+    # Creates a new experiment instance.
+    #
+    # The context provided to an experiment should be a consistent and unique
+    # value used to assign the same variant over many runs.
+    def initialize(context = {})
+      @context = context
+      @name = self.class.experiment_name
+      @run_id = SecureRandom.uuid
+      @run_key = run_key_hexdigest(context)
+      @options = {}
+    end
+
+    # Configures the experiment with the given options.
+    #
+    # This is used to set the variant, and can be used to set other options
+    # that may be used within an experiment. It's separate from the context to
+    # allow for the context to be used for variant assignment, while other
+    # options might still be useful.
+    #
+    # Raises an +ArgumentError+ if the variant is unknown.
+    # Returns self to allow chaining, typically for calling run.
+    def set(variant: nil, **options)
+      @options = @options.merge(options)
+      if variant.present?
+        variant = variant.to_sym
+        raise ArgumentError, "Unknown #{variant.inspect} variant" unless variants[variant]
+
+        @variant = variant
+      end
+
+      self
+    end
+
+    # Allows providing overrides for registered variants.
+    #
+    # When running experiments, any variant can be overridden to only invoke
+    # the provided override. This allows access to the scope and helpers where
+    # the experiment is being run. An example in a controller might look like:
+    #
+    #   MyExperiment.run(current_user) do |experiment|
+    #     experiment.on(:red) { render "red_pill" }
+    #     experiment.on(:blue) { redirect_to "blue_pill" }
+    #   end
+    #
+    # Raises an +ArgumentError+ if the variant is unknown, or if no block has
+    # been provided.
+    def on(*variant_names, &block)
+      variant_names.each do |variant|
+        variant = variant.to_sym
+        raise ArgumentError, "Unknown #{variant.inspect} variant" unless variants[variant]
+        raise ArgumentError, "Missing block" unless block
+
+        variant_step_chains[variant] = block
+      end
+    end
+
+    # Allows skipping the experiment.
+    #
+    # When an experiment is skipped, the default variant will be assigned, and
+    # generally means that no reporting should be generated for the run.
+    def skip
+      @skip = true
+    end
+
+    # Returns true if the experiment should be skipped.
+    #
+    # If the experiment has been instructed to be skipped manually, or if the
+    # rollout determines the experiment should be skipped.
+    def skipped?
+      return @skip if defined?(@skip)
+
+      @skip = self.rollout.skipped_for(self)
+    end
+
+    # Returns a hash with the experiment data.
+    #
+    # This is used to serialize the experiment data for logging and reporting,
+    # And can be overridden to provide additional data relevant to the
+    # experiment.
+    #
+    # Calling this before the variant has been assigned or resolved will result
+    # in the variant being empty. Generally, this should be called after the
+    # experiment has been run.
+    def serialize
+      {
+        "experiment" => name,
+        "run_id" => run_id,
+        "run_key" => run_key,
+        "variant" => variant.to_s,
+        "skipped" => skipped?
+      }
+    end
+
+    def to_s # :nodoc:
+      details = [@variant.inspect, @skip.inspect, (@run_key.slice(0, 16) + "..."), @context.inspect, @options.inspect]
+      string = "#<%s:%#0x @variant=%s @skip=%s @run_key=%s @context=%s, @options=%s>"
+      sprintf(string, self.class.name, object_id, *details)
+    end
+  end
+end