activeexperiment 0.1.0.alpha

data/lib/active_experiment/test_helper.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/class/subclasses"
+require "active_support/testing/assertions"
+
+module ActiveExperiment
+  # Provides helper methods for testing Active Experiments.
+  module TestHelper
+    include ActiveSupport::Testing::Assertions
+
+    def setup
+      super
+      clear_executed_experiments
+    end
+
+    def teardown
+      super
+      clear_executed_experiments
+    end
+
+    # Returns all of the experiments that have been executed.
+    def executed_experiments
+      ActiveExperiment::Executed.as_array
+    end
+
+    # Clears the list of executed experiments.
+    def clear_executed_experiments
+      ActiveExperiment::Executed.clear_all
+    end
+
+    # Provides the ability to stub an experiment's variant assignment, by
+    # changing the experiment to use the MockRollout within the scope of the
+    # provided block.
+    #
+    #   class SubjectExperiment < ActiveExperiment::Base
+    #     variant(:red) { "red" }
+    #     variant(:blue) { "blue" }
+    #     variant(:green) { "green" }
+    #   end
+    #
+    # Given the above is our experiment, the variant assignment can be stubbed
+    # to always assign the green variant.
+    #
+    #   stub_experiment(SubjectExperiment, :green) do
+    #     assert_equal "green", SubjectExperiment.run
+    #   end
+    #
+    # Or an array can be used to have green be assigned for the first run, and
+    # blue for the second, green for the third, and so on.
+    #
+    #   stub_experiment(SubjectExperiment, :green, :blue) do
+    #     assert_equal "green", SubjectExperiment.run
+    #     assert_equal "blue", SubjectExperiment.run
+    #     assert_equal "green", SubjectExperiment.run
+    #   end
+    #
+    # A lambda or proc can be used to provide custom variant assignment using
+    # whatever logic required.
+    #
+    #   resolver = lambda do |experiment|
+    #     experiment.context[:id] == 42 ? :green : :blue
+    #   end
+    #
+    #   stub_experiment(SubjectExperiment, resolver) do
+    #     assert_equal "blue", SubjectExperiment.run(id: 1)
+    #     assert_equal "blue", SubjectExperiment.run(id: 2)
+    #     assert_equal "green", SubjectExperiment.run(id: 42)
+    #   end
+    #
+    # Options can also be passed, for instance to simulate when an experiment
+    # should be skipped.
+    #
+    #   stub_experiment(SubjectExperiment, skip: true) do |mock_rollout|
+    #     assert_equal false, mock_rollout.skipped_for('_anything_')
+    #     assert_equal :red, mock_rollout.variant_for('_anything_')
+    #     assert_nil SubjectExperiment.run
+    #   end
+    #
+    # By default the +MockRollout+ class will be used, which implements all of
+    # the functionality described above -- however, any rollout class can be
+    # used for the span of the provided block.
+    #
+    #   stub_experiment(SubjectExperiment, rollout_class: MyCustomRollout) do
+    #     # ...
+    #   end
+    def stub_experiment(experiment_class, *variants, **options)
+      original_rollout = experiment_class.rollout
+
+      rollout_class = options.delete(:rollout_class) || MockRollout
+      rollout_options = { variant: variants }.merge(options)
+      experiment_class.rollout = rollout_class.new(experiment_class, **rollout_options)
+
+      _assert_nothing_raised_or_warn("stub_experiment") do
+        yield experiment_class.rollout
+      end
+    ensure
+      experiment_class.rollout = original_rollout
+    end
+
+    # Asserts that the number of experiments run matches the given number.
+    #
+    #   def test_experiments
+    #     assert_experiments 0
+    #     MyExperiment.run
+    #     assert_experiments 1
+    #     MyExperiment.run
+    #     assert_experiments 2
+    #   end
+    #
+    # If a block is provided, that block should cause the specified number of
+    # experiments to be run.
+    #
+    #   def test_experiments_again
+    #     assert_experiments 1 do
+    #       MyExperiment.run
+    #     end
+    #
+    #     assert_experiments 2 do
+    #       MyExperiment.run
+    #       MyExperiment.run
+    #     end
+    #   end
+    def assert_experiments(number, &block)
+      executed_count = executed_experiments.size
+      if block_given?
+        _assert_nothing_raised_or_warn("assert_experiments", &block)
+        executed_count = executed_experiments.size - executed_count
+        assert_equal number, executed_count, "#{number} experiment runs expected, but found #{executed_count}"
+      else
+        assert_equal number, executed_count
+      end
+    end
+
+    # Asserts that no experiments have been run.
+    #
+    #   def test_experiments
+    #     assert_no_experiments
+    #     MyExperiment.run
+    #     assert_experiments 1
+    #   end
+    #
+    # If a block is passed, that block should not cause any emails to be sent.
+    #
+    #   def test_experiments_again
+    #     assert_no_experiments do
+    #       # No experiments should be run from this block
+    #     end
+    #   end
+    #
+    # Note: This assertion is simply a shortcut for:
+    #
+    #   assert_experiments(0, &block)
+    def assert_no_experiments(&block)
+      assert_experiments(0, &block)
+    end
+
+    # Asserts that a specific experiment has been run, optionally matching args
+    # and/or context.
+    #
+    #   def test_experiment
+    #     MyExperiment.run(id: 1)
+    #     assert_experiment_with MyExperiment, context: { id: 1 }
+    #   end
+    #
+    #   def test_experiment_with_options
+    #     MyExperiment.set(foo: :bar).run(id: 1)
+    #     assert_experiment_with MyExperiment, options: { foo: "bar" }
+    #   end
+    #
+    #   def test_experiment_with_variant
+    #     MyExperiment.set(variant: :red).run(id: 1)
+    #     assert_experiment_with MyExperiment, variant: :red
+    #   end
+    #
+    # If a block is passed, that block should cause the specified experiment to
+    # be run.
+    #
+    #   def test_experiment_in_block
+    #     assert_experiment_with MyExperiment, context: { id: 1 } do
+    #       MyExperiment.run(id: 1)
+    #     end
+    #   end
+    def assert_experiment_with(experiment_class, context: nil, options: nil, variant: nil, &block)
+      expected = { context: context, options: options, variant: variant }.compact
+      experiments = executed_experiments
+      if block_given?
+        original_executed_experiments = experiments.dup
+        _assert_nothing_raised_or_warn("assert_experiment_with", &block)
+        experiments = executed_experiments - original_executed_experiments
+      end
+
+      match_potential = []
+      match_class = []
+      match_experiment = experiments.find do |experiment|
+        match_potential << experiment
+        if experiment.class == experiment_class
+          match_class << experiment
+          expected.all? do |key, value|
+            experiment.public_send(key) == value
+          end
+        end
+      end
+
+      message = +"No matching run found for #{experiment_class.name}"
+      message << " with #{expected.inspect}" if expected.any?
+      if match_potential.empty?
+        message << "\n\nNo experiment were run"
+      elsif match_class.empty?
+        message << "\n\nNo #{experiment_class.name} experiments were run, experiments run:"
+        message << "\n  #{match_potential.map(&:class).join(", ")}"
+      else
+        message << "\n\nPotential matches:"
+        message << "\n  #{match_class.join("\n  ")}"
+      end
+
+      assert(match_experiment, message)
+      match_experiment
+    end
+
+    # Mock rollout class that can be used to stub out the variant assignment of
+    # an experiment.
+    #
+    # This is used by the ActiveExperiment::TestHelper in the +stub_experiment+
+    # method. It can be used directly, or through the helper when needing to
+    # stub out the rollout of an experiment in a test. It can also be inherited
+    # and customized.
+    class MockRollout < Rollouts::BaseRollout
+      def initialize(...)
+        @assigned = 0
+        super
+      end
+
+      def skipped_for(ex)
+        raise ArgumentError, "expecting a #{@experiment_class.name}" unless ex.is_a?(@experiment_class)
+
+        skip = opts[:skip]
+
+        # Accepts a callable in the :skip option.
+        return skip.call(ex) if skip.respond_to?(:call)
+
+        # Accepts a boolean in the :skip options, with a default of false.
+        !!skip
+      end
+
+      def variant_for(ex)
+        raise ArgumentError, "expecting a #{@experiment_class.name}" unless ex.is_a?(@experiment_class)
+
+        # Accepts an array in the :variant option.
+        variant = opts[:variant]
+        variant = variant[((@assigned += 1) - 1) % variant.size] if variant.is_a?(Array) && !variant.empty?
+
+        # Accepts a callable in the :variant option.
+        return variant.call(ex) if variant.respond_to?(:call)
+
+        # Accepts a symbol in the :variant option.
+        return variant if variant.is_a?(Symbol)
+
+        # Fall back to the default variant.
+        ex.try(:default_variant)
+      end
+
+      def opts
+        @rollout_options
+      end
+    end
+  end
+end

data/lib/active_experiment/variants.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Variant Registration
+  #
+  # Variants are registered using the +control+ and +variant+ methods within an
+  # experiment. The +control+ method is a convenience for registering a variant
+  # with the name +:control+ -- a convention used to describe the default
+  # variant.
+  #
+  # Registering a variant can be done by providing a name, and a block or a
+  # symbol referencing a method.
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     control { "control" } # defines a variant named :control
+  #     variant :treatment, :treatment_method
+  #
+  #     private
+  #
+  #     def treatment_method
+  #       "treatment"
+  #     end
+  #   end
+  #
+  # Subclassing experiments will inherit variants. Existing variants can be
+  # overridden or added to, and new variants can registered.
+  #
+  #   class NewExperiment < MyExperiment
+  #     control(override: true) { "new control" }
+  #     variant(:treatment, add: true, prepend: true) { "new treatment" }
+  #     variant(:new_variant) { "new variant" }
+  #   end
+  #
+  # In the above example, the control is overridden, a new variant is added,
+  # and a new step is added to the treatment variant.
+  #
+  # When running an experiment, variants can be overridden by using the +on+
+  # method. This allows utilizing the scope of where the experiment is run to
+  # change the experiment behavior.
+  #
+  #   NewExperiment.run do |experiment|
+  #     experiment.on(:treatment) { "overridden treatment" }
+  #   end
+  #
+  # By default, the "control" variant is assigned as the default variant, but
+  # any variant can be specified as the default. The concept of the control
+  # variant is only a convention.
+  #
+  # To specify a different default variant:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     use_default_variant :blue
+  #   end
+  #
+  # The default variant is assigned if the experiment is skipped or if no other
+  # variant has been resolved after asking the rollout -- when the rollout may
+  # not be working properly.
+  module Variants
+    extend ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    VARIANT_CHAIN_SUFFIX = "_variant"
+    private_constant :VARIANT_CHAIN_SUFFIX
+
+    STEP_CHAIN_SUFFIX = "_steps"
+    private_constant :STEP_CHAIN_SUFFIX
+
+    included do
+      class_attribute :variants, instance_writer: false, instance_predicate: false, default: {}
+      class_attribute :default_variant, instance_writer: false, instance_predicate: false, default: :control
+    end
+
+    # These methods will be included into any Active Experiment object, adding
+    # the the ability to set the default variant, defining variants and their
+    # behaviors and callback helpers.
+    module ClassMethods
+      private
+        def use_default_variant(variant)
+          variant = variant.to_sym
+          raise ArgumentError, "Unknown #{variant.inspect} variant" unless variants[variant]
+
+          self.default_variant = variant
+        end
+
+        def register_variant_callback(variant, *filters, override: false, add: false, **options, &block)
+          raise ArgumentError, "Provide either `override: true` or `add: true` but not both" if override && add
+
+          variant = variant.to_sym
+          if variants[variant].present?
+            unless override || add
+              raise ArgumentError, "The #{variant.inspect} variant is already registered. " \
+                "Provide `override: true` or `add: true` to make changes to it."
+            end
+          elsif override || add
+            raise ArgumentError, "Unable to override or add to unknown #{variant.inspect} variant"
+          end
+
+          self.variants = variants.dup unless singleton_class.method_defined?(:variants, false)
+
+          variants[variant] = callback_chain = :"#{variant}#{VARIANT_CHAIN_SUFFIX}"
+
+          unless add
+            define_variant_callbacks(callback_chain) # variant callback chain
+            define_variant_callbacks("#{callback_chain}#{STEP_CHAIN_SUFFIX}") # variant step chain
+          end
+
+          filters.push(block) if block.present?
+          filters.unshift(callback_chain) if filters.empty?
+          set_callback_with_target("#{callback_chain}#{STEP_CHAIN_SUFFIX}", *filters, **options) do |target, callback|
+            target.instance_variable_set(:@results, callback.call(target, nil))
+          end
+        end
+
+        def define_variant_callbacks(callback_chain)
+          define_callbacks(callback_chain)
+          private :"_#{callback_chain}_callbacks", :"_run_#{callback_chain}_callbacks"
+        end
+
+        def set_variant_callback(variant, type, *filters, &block)
+          raise ArgumentError, "Unknown `#{variant}` variant" unless variants[variant.to_sym]
+
+          set_callback("#{variant}#{VARIANT_CHAIN_SUFFIX}", type, *filters, &block)
+        end
+    end
+
+    # The names for all registered variants.
+    #
+    # This is most commonly used by rollouts, where knowing the variant names
+    # is important for determining which variant to assign.
+    def variant_names
+      variants.keys
+    end
+
+    private
+      def variant_step_chains
+        @variant_step_chains ||= variants.transform_values do |callback_chain|
+          chain_name = "#{callback_chain}#{STEP_CHAIN_SUFFIX}"
+          -> { run_callbacks(chain_name, :process_variant_steps) { @results } }
+        end
+      end
+  end
+end

data/lib/active_experiment/version.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "gem_version"
+
+module ActiveExperiment
+  # Returns the currently loaded version of Active Experiment as a
+  # +Gem::Version+.
+  def self.version
+    gem_version
+  end
+end

data/lib/active_experiment.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "global_id"
+require "active_support"
+require "active_support/rails"
+require "active_support/tagged_logging"
+
+require "active_experiment/version"
+
+module ActiveExperiment
+  Error = Class.new(StandardError)
+  ExecutionError = Class.new(Error)
+
+  extend ActiveSupport::Autoload
+
+  autoload :Base
+  autoload :Cache
+  autoload :ConfiguredExperiment
+  autoload :Executed
+  autoload :Rollouts
+  autoload :Capturable
+
+  autoload :TestCase
+  autoload :TestHelper
+
+  mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(STDOUT))
+end

data/lib/activeexperiment.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# This file is here to make the gem behave like a Rails core library...
+require "active_experiment"
+
+# But we'll also require the railtie if it looks like it's loading in a Rails
+# app, to avoid having to require it manually in application.rb.
+require "active_experiment/railtie" if defined?(Rails)

data/lib/rails/generators/experiment/USAGE

@@ -0,0 +1,12 @@
+Description:
+    Generates a new experiment. Pass the experiment name, either CamelCased or
+    under_scored, with or without the experiment postfix. Optionally, provide a
+    list of variant names.
+
+Examples:
+    `bin/rails generate experiment RedBlueExperiment red blue`
+
+      Creates the the following files:
+
+        Experiment: app/experiments/red_blue_experiment.rb
+        Test:       test/experiments/red_blue_experiment_test.rb

data/lib/rails/generators/experiment/experiment_generator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+
+module Rails # :nodoc:
+  module Generators # :nodoc:
+    class ExperimentGenerator < Rails::Generators::NamedBase # :nodoc:
+      class_option :parent, type: :string, default: "ApplicationExperiment", desc: "The parent class for the generated experiment"
+      class_option :skip_comments, type: :boolean, default: false, desc: "Omit helpful comments from generated files"
+
+      argument :variants, type: :array, default: %w[control treatment], banner: "variant variant"
+
+      check_class_collision suffix: "Experiment"
+
+      hook_for :test_framework
+
+      def self.default_generator_root
+        __dir__
+      end
+
+      def create_experiment_file
+        template "experiment.rb", File.join("app/experiments", class_path, "#{file_name}_experiment.rb")
+
+        in_root do
+          if behavior == :invoke && !File.exist?(application_experiment_file_name)
+            template "application_experiment.rb", application_experiment_file_name
+          end
+        end
+      end
+
+      private
+        def parent_class_name
+          options[:parent]
+        end
+
+        def variant_names
+          @variant_names ||= variants.map { |variant| variant.to_s.underscore }
+        end
+
+        def file_name
+          @_file_name ||= super.sub(/_experiment\z/i, "")
+        end
+
+        def application_experiment_file_name
+          @application_experiment_file_name ||= if mountable_engine?
+            "app/experiments/#{namespaced_path}/application_experiment.rb"
+          else
+            "app/experiments/application_experiment.rb"
+          end
+        end
+    end
+  end
+end

data/lib/rails/generators/experiment/templates/application_experiment.rb.tt

@@ -0,0 +1,4 @@
+<% module_namespacing do -%>
+class ApplicationExperiment < ActiveExperiment::Base
+end
+<% end -%>

data/lib/rails/generators/experiment/templates/experiment.rb.tt

@@ -0,0 +1,35 @@
+<% module_namespacing do -%>
+class <%= class_name %>Experiment < <%= parent_class_name.classify %>
+<% variant_names.each do |variant| -%>
+<% if variant == "control" -%>
+  <%= variant -%> { }
+<% else -%>
+  variant(:<%= variant -%>) { }
+<% end -%>
+<% end -%>
+
+<% unless options[:skip_comments] -%>
+<% unless variant_names.include?("control") -%>
+  # Specify a default variant to use when the experiment is skipped:
+  #use_default_variant :<%= variant_names.first %>
+  #
+<% end -%>
+  # Run this experiment by providing a context (current_user in this example),
+  # and optionally override the variant behaviors:
+  #
+  #   <%= class_name %>Experiment.run(current_user) do |experiment|
+<% variant_names.each do |variant| -%>
+  #     experiment.on(:<%= variant -%>) { "overridden <%= variant -%>" }
+<% end -%>
+  #   end
+  #
+  # Each context (user) will consistently have the same variant assigned. More
+  # advanced logic can be provided to segment contexts into specific variants:
+  #
+  #segment :old_accounts, into: :<%= variant_names.last %>
+  #def old_accounts
+  #  context.created_at < 1.year.ago
+  #end
+<% end -%>
+end
+<% end -%>

data/lib/rails/generators/rspec/experiment/experiment_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# require "generators/rspec"
+
+module Rspec # :nodoc:
+  module Generators # :nodoc:
+    class ExperimentGenerator < Rails::Generators::NamedBase # :nodoc:
+      source_root(File.expand_path("templates", __dir__))
+
+      def create_spec_file
+        template "experiment_spec.rb", File.join("spec/experiments", class_path, "#{file_name}_experiment_spec.rb")
+      end
+
+      private
+        def file_name
+          @_file_name ||= super.sub(/_experiment\z/i, "")
+        end
+    end
+  end
+end

data/lib/rails/generators/rspec/experiment/templates/experiment_spec.rb.tt

@@ -0,0 +1,7 @@
+require "rails_helper"
+
+<% module_namespacing do -%>
+RSpec.describe <%= class_name -%>Experiment, type: :experiment do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/rails/generators/test_unit/experiment/experiment_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "rails/generators/test_unit"
+
+module TestUnit # :nodoc:
+  module Generators # :nodoc:
+    class ExperimentGenerator < Base # :nodoc:
+      check_class_collision suffix: "ExperimentTest"
+      source_root(File.expand_path("templates", __dir__))
+
+      def create_test_file
+        template "experiment_test.rb", File.join("test/experiments", class_path, "#{file_name}_experiment_test.rb")
+      end
+
+      private
+        def file_name
+          @_file_name ||= super.sub(/_experiment\z/i, "")
+        end
+    end
+  end
+end

data/lib/rails/generators/test_unit/experiment/templates/experiment_test.rb.tt

@@ -0,0 +1,9 @@
+require "test_helper"
+
+<% module_namespacing do -%>
+class <%= class_name %>ExperimentTest < ActiveJob::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end
+<% end -%>