activeexperiment 0.1.0.alpha

data/lib/active_experiment/executed.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "active_support/current_attributes"
+
+module ActiveExperiment
+  # == Executed Experiments
+  #
+  # This is useful for surfacing experiments to the client layer. For example,
+  # if you have a client only experiment, an empty experiment can be defined
+  # and then run in the controller layer. The client layer can then use the
+  # experiment information to render the appropriate client code based on the
+  # variant that's been assigned, or other information available the experiment
+  # can provide in its +serialize+ method.
+  #
+  # An example of an empty experiment might be as simple as:
+  #
+  #   class MyClientExperiment < ActiveExperiment::Base
+  #     variant(:red) { }
+  #     variant(:blue) { }
+  #   end
+  #
+  # In the controller, this experiment can be run in a before action, or
+  # anywhere else that makes sense for your application:
+  #
+  #   class MyController < ApplicationController
+  #     before_action :run_my_client_experiment
+  #     # ... controller code
+  #
+  #     private
+  #
+  #     def run_my_client_experiment
+  #       MyClientExperiment.run(current_user)
+  #     end
+  #   end
+  #
+  # Then in the layout, or appropriate view, all experiments that have been run
+  # during the request can be surfaced:
+  #
+  #   <title>My App</title>
+  #   <script>
+  #     window.experiments = <%= ActiveExperiment::Executed.to_json %>
+  #   </script>
+  #
+  # Or the experiments that have been run can be iterated:
+  #
+  #   <% ActiveExperiment::Executed.experiments.each do |experiment| %>
+  #     <meta name="<%= experiment.name %>" content="<%= experiment.variant %>">
+  #   <% end %>
+  class Executed < ActiveSupport::CurrentAttributes
+    attribute :experiments
+
+    # Interface to add an experiment to the executed experiments. This is
+    # intended to be used by the +run+ method of the experiment class.
+    #
+    # Experiments are added to the executed experiments if they have been
+    # assigned a variant.
+    def self.<<(experiment)
+      self.experiments ||= []
+      experiments << experiment
+    end
+
+    # Returns an array of experiments that have been run.
+    #
+    # Used by several other methods to return serialized experiments.
+    def self.as_array
+      self.experiments || []
+    end
+
+    # Returns a json of the experiments that have been run, with the experiment
+    # name as the key, and the serialized experiment as the value. This assumes
+    # that if an experiment is run multiple times, the same variant has been
+    # assigned for all runs, which may not be true, like when using the random
+    # rollout.
+    #
+    # When needed, the executed experiments can be accessed and/or iterated
+    # directly, or the +to_json_array+ method can be used.
+    def self.to_json
+      as_array.each_with_object({}) { |e, hash| hash[e.name] = e.serialize }.to_json
+    end
+
+    # Returns an array of experiments that have been run.
+    def self.to_json_array
+      as_array.map(&:serialize).to_json
+    end
+  end
+end

data/lib/active_experiment/execution.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Execution Module
+  #
+  # This module provides most of the logic for running experiments. Running an
+  # experiment can be performed in a few ways, some of which are provided as
+  # convenience.
+  #
+  # 1. Calling +run+ on the class, passing the context and a block:
+  #
+  #   MyExperiment.run(context) do |experiment|
+  #     experiment.on(:treatment) { "treatment" }
+  #   end
+  #
+  # 2. Instantiating the experiment with the context, and calling +run+:
+  #
+  #   MyExperiment.new(context).run do |experiment|
+  #     experiment.on(:treatment) { "treatment" }
+  #   end
+  #
+  # 3. Using the +ConfiguredExperiment+ API to +set+ and then +run+:
+  #
+  #   MyExperiment.set(variant: :treatment).run(id: 1) do |experiment|
+  #     experiment.on(:treatment) { "treatment" }
+  #   end
+  #
+  # In all cases, a block can be provided to the +run+ method. The block will
+  # be called with the experiment, which allows overriding the variant
+  # behaviors using the scope of where the experiment is being run.
+  #
+  # When the experiment is run, the variant will be determined and the variant
+  # steps will be executed. The result of the variant execution will be
+  # returned unless the experiment is aborted in a +before_run+ or
+  # +before_variant+ callback.
+  #
+  # In general, the following decision tree diagram helps illustrate the order
+  # that things will be executed in running an experiment, utilizing caching
+  # when possible:
+  #                             run
+  #                              |
+  #                         _ skipped? _
+  #                        |            |
+  #                        no          yes
+  #                        |            |
+  #                        |    assigned/default_variant
+  #                        |
+  #               _ cached_variant? _
+  #              |                   |
+  #              no                 yes
+  #              |                   |
+  #       _ segmented? _      (cached value)
+  #      |              |
+  #     yes             no
+  #      |              |
+  #      |  ___ rollout.variant_for __
+  #      | |            |             |
+  #    (cache)       (cache)       (cache)
+  #   variant_a     variant_b     variant_c
+  #
+  module Execution
+    extend ActiveSupport::Concern
+
+    # These methods will be included into any Active Experiment object and
+    # expose the class level run method, and the ability to get a configured
+    # experiment instance using the set method.
+    module ClassMethods
+      # Instantiates and runs an experiment with the provided context and
+      # block. This is a convenience method.
+      #
+      # An example of using this method to run an experiment:
+      #
+      #   MyExperiment.run(id: 1) do |experiment|
+      #     experiment.on(:treatment) { "red" }
+      #   end
+      def run(*args, **kws, &block)
+        new(*args, **kws).run(&block)
+      end
+
+      # Creates a configured experiment with the provided options. Configured
+      # experiments expose a few helpful methods for running and caching
+      # experiment details.
+      #
+      # The following options can be provided to configure an experiment:
+      #
+      # * +:variant+ - The variant to assign.
+      #
+      # An example of using this method to set a variant and run an experiment:
+      #
+      #   MyExperiment.set(variant: :red).run(id: 1) do |experiment|
+      #     experiment.on(:red) { "red" }
+      #   end
+      def set(**options)
+        ConfiguredExperiment.new(self, **options)
+      end
+    end
+
+    # Runs the experiment. Calling +run+ returns the value of the assigned
+    # variant block or method.
+    #
+    # When running an experiment, a block can be provided and it will be called
+    # with the experiment, which provides the ability to override variant
+    # behaviors when running the experiment.
+    #
+    #   MyExperiment.new(id: 1).run do |experiment|
+    #     experiment.on(:treatment) { "treatment" }
+    #   end
+    #
+    # Raises an ActiveExperiment::ExecutionError if there are no variants
+    # registered, or if the experiment is already running, in the case of
+    # accidentally calling run again within a run or variant block.
+    def run(&block)
+      return @results if defined?(@results)
+      raise ExecutionError, "No variants registered" if variant_names.empty?
+
+      @results = nil
+      instrument(:start_experiment)
+      instrument(:process_run) do
+        run_callbacks(:run, :process_run_callbacks) do
+          call_run_block(&block) if block.present?
+          @variant = resolve_variant
+          @results = resolve_results
+        end
+      end
+
+      @results
+    ensure
+      Executed << self
+    end
+
+    private
+      def resolve_variant
+        return variant || default_variant if skipped?
+
+        resolved = cached_variant(variant) do
+          run_callbacks(:segment, :process_segment_callbacks)
+          variant || rollout.variant_for(self)
+        end
+
+        variant || resolved || default_variant
+      end
+
+      def resolve_results
+        resolved = nil
+        run_callbacks(variants[variant], :process_variant_callbacks) do
+          resolved = variant_step_chains[variant]&.call
+        end
+
+        resolved || @results
+      end
+
+      def call_run_block(&block)
+        block.call(self)
+      end
+  end
+end

data/lib/active_experiment/gem_version.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # Returns the currently loaded version of Active Experiment as a
+  # +Gem::Version+.
+  def self.gem_version
+    Gem::Version.new(VERSION::STRING)
+  end
+
+  module VERSION
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+    PRE   = "alpha"
+
+    STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")
+  end
+end

data/lib/active_experiment/instrumentation.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Instrumentation Module
+  #
+  # Instrumentation is provided through +ActiveSupport::Notifications+. The
+  # best example of how to utilize the instrumentation within Active Experiment
+  # is look at how +ActiveExperiment::LogSubscriber+ has been implemented.
+  module Instrumentation
+    extend ActiveSupport::Concern
+
+    private
+      def run_callbacks(kind, event_name, **payload, &block)
+        if kind.present? && __callbacks[kind.to_sym].any?
+          instrument(event_name, **payload) do
+            super(kind, &block)
+          end
+        else
+          yield if block_given?
+        end
+      end
+
+      def instrument(operation, **payload, &block)
+        operation = "#{operation}.active_experiment"
+        payload = payload.merge(experiment: self)
+        return ActiveSupport::Notifications.instrument(operation, payload) unless block.present?
+
+        ActiveSupport::Notifications.instrument(operation, payload) do |event_payload|
+          variant = @variant
+          results = block.call
+
+          event_payload[:variant] = @variant if variant != @variant
+          event_payload[:aborted] = @halted_callback if @halted_callback.present?
+          @halted_callback = nil if @halted_callback == :segment
+
+          results
+        end
+      end
+
+      def halted_callback_hook(filter, name)
+        super
+        @halted_callback = name
+      end
+  end
+end

data/lib/active_experiment/log_subscriber.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "active_support/log_subscriber"
+
+module ActiveExperiment
+  # == Log Subscriber
+  #
+  # TODO: finish documenting.
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def start_experiment(event)
+      warn_of_nested_experiment(event) if execution_stack.any?
+      experiment_logger(event) do |experiment|
+        execution_stack.push(experiment)
+
+        info = []
+        info << "Run ID: #{experiment.run_id}"
+        info << "Variant: #{experiment.variant}" if experiment.variant.present?
+
+        build_message(:info, "Running #{experiment.name} (#{info.join(", ")})", context: experiment.log_context?)
+      end
+    end
+
+    def process_run(event)
+      errored = event.payload[:exception_object]
+      aborted = !errored && event.payload[:aborted]
+
+      experiment_logger(event) do |experiment|
+        execution_stack.pop
+
+        if errored
+          build_message(:error, "Run failed: #{errored.class} (#{errored.message})")
+        elsif aborted
+          build_message(:info, "Run aborted in #{aborted} callbacks", details: true)
+        else
+          variant_name = experiment.variant
+          if experiment.variant_names.include?(variant_name)
+            build_message(:info, "Completed running #{experiment.variant} variant", details: true)
+          elsif variant_name.present?
+            build_message(:error, "Run errored: unknown `#{variant_name}` variant resolved", details: true)
+          else
+            build_message(:error, "Run errored: no variant resolved", details: true)
+          end
+        end
+      end
+    end
+
+    def process_segment_callbacks(event)
+      return if event.payload[:exception_object].present?
+
+      experiment_logger(event) do |experiment|
+        if event.payload[:aborted] == :segment
+          build_message(:info, "Segmented into the `#{experiment.variant}` variant", duration: true)
+        else
+          build_callback_message(event)
+        end
+      end
+    end
+
+    def process_run_callbacks(event)
+      experiment_logger(event) { build_callback_message(event) }
+    end
+
+    def process_variant_callbacks(event)
+      experiment_logger(event) { build_callback_message(event) }
+    end
+
+    def process_variant_steps(event)
+      experiment_logger(event) { build_callback_message(event) }
+    end
+
+    private
+      def warn_of_nested_experiment(event)
+        experiment_logger(event) do
+          build_message(:warn, "Nesting experiment in #{experiment_identifier(execution_stack.last)}")
+        end
+      end
+
+      def experiment_logger(event, &block)
+        return unless logger.present?
+
+        experiment = event.payload[:experiment]
+        result = block.call(experiment)
+
+        return unless result.present?
+
+        logger.send(result[:level]) do
+          log = +colorized_prefix(experiment)
+          log << colorized_message(result[:message], level: result[:level])
+          log << colorized_duration(event, parens: true) if result[:duration]
+          log << colorized_details(event) if result[:details]
+          log << colorized_context(experiment) if result[:context]
+          log
+        end
+      end
+
+      def build_message(level, message, **kws)
+        { level: level, message: message, **kws }
+      end
+
+      def build_callback_message(event)
+        return if event.payload[:exception_object].present?
+
+        variant = event.payload[:variant]
+        process = event.name.split(".").first.gsub("process_", "").tr("_", " ")
+
+        if variant.present? && process != "run callbacks"
+          build_message(:info, "Resolved `#{variant}` variant in #{process}", duration: true)
+        elsif process != "variant steps"
+          build_message(:debug, "Completed #{process}", duration: true)
+        end
+      end
+
+      def colorized_prefix(experiment)
+        color("  #{experiment_identifier(experiment)}  ", GREEN)
+      end
+
+      def colorized_message(message, level: :info)
+        case level
+        when :error
+          color(message, RED, bold: true)
+        when :warn
+          color(message, YELLOW, bold: true)
+        else
+          message
+        end
+      end
+
+      def colorized_details(event)
+        " (Duration:#{colorized_duration(event, parens: false)} | Allocations: #{event.allocations})"
+      end
+
+      def colorized_duration(event, parens: true)
+        duration = event.duration.round(1)
+        if duration > 1000
+          ret = color("#{duration}ms", RED, bold: true)
+        elsif duration > 500
+          ret = color("#{duration}ms", YELLOW, bold: true)
+        else
+          ret = "#{duration}ms"
+        end
+
+        parens ? " (#{ret})" : " #{ret}"
+      end
+
+      def colorized_context(experiment)
+        return "" unless experiment.log_context?
+
+        " with context: #{format_context(experiment.context).inspect}"
+      end
+
+      def format_context(arg)
+        case arg
+        when Hash
+          arg.transform_values { |value| format_context(value) }
+        when Array
+          arg.map { |value| format_context(value) }
+        when GlobalID::Identification
+          arg.to_global_id.to_s rescue arg
+        else
+          arg
+        end
+      end
+
+      def experiment_identifier(experiment)
+        "#{experiment.class.name}[#{experiment.run_key.slice(0, 8)}]"
+      end
+
+      def execution_stack
+        ActiveSupport::IsolatedExecutionState[:active_experiment_log_subscriber_execution_stack] ||= []
+      end
+
+      def logger
+        ActiveExperiment.logger
+      end
+  end
+end
+
+ActiveExperiment::LogSubscriber.attach_to(:active_experiment)

data/lib/active_experiment/logging.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Logging Module
+  #
+  # Within experiments, you can log information about the experiment. This can
+  # be done by simply calling +logger+ within the experiment definition. By
+  # default the logger will be tagged with "ActiveExperiment" to help identify
+  # where the log message is coming from.
+  #
+  # The logger can also be tagged, and then used from within a block. The tag
+  # will be removed after the block has been executed.
+  #
+  # For instance, an experiment could implement tag the logger like this when
+  # run:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     around_run(prepend: true) do |_, block|
+  #       tag_logger("MyExperiment", "run", &block)
+  #     end
+  #   end
+  #
+  # Any logging that occurs within this experiment, while it's being run, will
+  # be tagged with "MyExperiment" and "run". Those log lines might look like:
+  #
+  #   "[ActiveExperiment] [MyExperiment] [run] Experiment started..."
+  #
+  # The +log_context+ class attribute is used as configuration within the
+  # +ActiveExperiment::LogSubscriber+. For some experiments that contain
+  # sensitive information, it might be useful to not log the context. This can
+  # be done by setting that experiment class's +log_context+ to false.
+  #
+  # More logging details can be found in the +ActiveExperiment::LogSubscriber+.
+  module Logging # :nodoc:
+    extend ActiveSupport::Concern
+
+    TAG_NAME = "ActiveExperiment"
+    private_constant :TAG_NAME
+
+    included do
+      class_attribute :log_context, instance_predicate: true, default: false
+      private :log_context=, :log_context
+    end
+
+    # Returns logger that can be used within the experiment.
+    #
+    # The logger will be tagged with "ActiveExperiment" if possible, to help
+    # identify where the log messages are coming from.
+    def logger
+      @logger ||= ActiveExperiment.logger.try(:tagged, TAG_NAME) || ActiveExperiment.logger
+    end
+
+    private
+      def tag_logger(*tags, &block)
+        if logger.respond_to?(:tagged)
+          logger.tagged(*tags, &block)
+        else
+          yield
+        end
+      end
+  end
+end

data/lib/active_experiment/railtie.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "global_id/railtie"
+require "active_experiment"
+
+module ActiveExperiment
+  class Railtie < Rails::Railtie # :nodoc:
+    config.active_experiment = ActiveSupport::OrderedOptions.new
+    config.active_experiment.custom_rollouts = {}
+    config.active_experiment.log_query_tags_around_run = true
+
+    initializer "active_experiment.logger" do
+      ActiveSupport.on_load(:active_experiment) { ActiveExperiment.logger = ::Rails.logger }
+    end
+
+    initializer "active_experiment.custom_rollouts" do |app|
+      config.after_initialize do
+        app.config.active_experiment.custom_rollouts.each do |name, rollout|
+          ActiveExperiment::Rollouts.register(name, rollout)
+        end
+      end
+    end
+
+    initializer "active_experiment.set_configs" do |app|
+      options = app.config.active_experiment
+      config.after_initialize do
+        options.digest_secret_key ||= app.secrets.secret_key_base
+
+        options.each do |k, v|
+          k = "#{k}="
+          if ActiveExperiment.respond_to?(k)
+            ActiveExperiment.send(k, v)
+          end
+        end
+      end
+
+      ActiveSupport.on_load(:active_experiment) do
+        options.each do |k, v|
+          k = "#{k}="
+          if ActiveExperiment.respond_to?(k)
+            ActiveExperiment.send(k, v)
+          elsif respond_to?(k)
+            send(k, v)
+          end
+        end
+      end
+
+      ActiveSupport.on_load(:action_dispatch_integration_test) do
+        include ActiveExperiment::TestHelper
+      end
+    end
+
+    initializer "active_experiment.query_log_tags" do |app|
+      query_logs_tags_enabled = app.config.respond_to?(:active_record) &&
+        app.config.active_record.query_log_tags_enabled &&
+        app.config.active_experiment.log_query_tags_around_run
+
+      if query_logs_tags_enabled
+        app.config.active_record.query_log_tags |= [:experiment]
+
+        ActiveSupport.on_load(:active_record) do
+          ActiveRecord::QueryLogs.taggings[:experiment] = lambda do |context|
+            context[:experiment].class.name if context[:experiment]
+          end
+        end
+      end
+    end
+  end
+end