activeexperiment 0.1.0.alpha

data/lib/active_experiment/rollout.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Rollout Module
+  #
+  # Active Experiment can be configured to use one of the built in rollouts, or
+  # a custom rollout.
+  #
+  # When configuring the default rollout, you can use a symbol or something
+  # that responds to the required +skipped_for+ and +variant_for+ methods. To
+  # configure the default rollout:
+  #
+  #   ActiveExperiment::Base.default_rollout = :random
+  #
+  # The above example will use the built in +:random+ rollout for all
+  # experiments. A given experiment can also be configured to use a specific
+  # rollout which will override the default:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     rollout :percent, rules: { blue: 60, red: 40 }
+  #   end
+  #
+  # An experiment might even be configured to use itself as a rollout. As
+  # long as the class responds to the required methods, it can be used.
+  #
+  #   module ExperimentRolloutExample
+  #     def skipped_for(*)
+  #       false
+  #     end
+  #
+  #     def variant_for(*)
+  #       :red
+  #     end
+  #   end
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     extend ExperimentRolloutExample
+  #
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     use_rollout self
+  #   end
+  #
+  # The flexibility that this affords is that you can use any rollout you want
+  # for any experiment. Rollouts can be defined that use a database, feature
+  # flags, or any other mechanism you want.
+  module Rollout
+    extend ActiveSupport::Concern
+
+    REQUIRED_ROLLOUT_METHODS = [:skipped_for, :variant_for].freeze
+    private_constant :REQUIRED_ROLLOUT_METHODS
+
+    included do
+      class_attribute :rollout, instance_predicate: false
+      private :rollout=
+
+      self.default_rollout = :percent
+    end
+
+    # These methods will be included into any Active Experiment object and
+    # allow setting the default rollout, the experiment specific rollout, and
+    # includes the inherited behavior for default rollouts. When setting the
+    # rollout, it will be validated to ensure it responds to the required
+    # methods.
+    module ClassMethods
+      def inherited(subclass) # :nodoc:
+        super
+        subclass.default_rollout = @default_rollout
+      end
+
+      # Allows setting the default rollout for all experiments.
+      #
+      # This can be overridden on a per experiment basis, but overrides to the
+      # default rollout are not be inherited. Meaning that each experiment will
+      # revert to the default rollout, regardless of what it inherits from.
+      def default_rollout=(name_or_rollout)
+        use_rollout(name_or_rollout)
+        @default_rollout = name_or_rollout
+      end
+
+      private
+        def use_rollout(name_or_rollout, *args, **kws, &block)
+          case name_or_rollout
+          when Symbol, String
+            rollout = ActiveExperiment::Rollouts.lookup(name_or_rollout)
+            self.rollout = rollout.new(self, *args, **kws, &block)
+          else
+            unless rollout_interface?(name_or_rollout)
+              raise ArgumentError, "Invalid rollout. " \
+                "Rollouts must respond to #{REQUIRED_ROLLOUT_METHODS.join(", ")}."
+            end
+
+            self.rollout = name_or_rollout
+          end
+        end
+
+        def rollout_interface?(object)
+          REQUIRED_ROLLOUT_METHODS.all? { |meth| object.respond_to?(meth) }
+        end
+    end
+  end
+end

data/lib/active_experiment/rollouts/inactive_rollout.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  module Rollouts
+    # == Active Experiment Inactive Rollout
+    #
+    # Using this rollout will disable experiments as though they were
+    # intentionally skipped.
+    #
+    # To use as the default, configure it to +:inactive+.
+    #
+    #   ActiveExperiment::Base.default_rollout = :inactive
+    #   Rails.application.config.active_experiment.default_rollout = :inactive
+    class InactiveRollout < BaseRollout
+      def skipped_for(*)
+        true
+      end
+
+      def variant_for(*)
+        nil
+      end
+    end
+  end
+end

data/lib/active_experiment/rollouts/percent_rollout.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module ActiveExperiment
+  module Rollouts
+    # == Active Experiment Percent Rollout
+    #
+    # The percent rollout is the most comprehensive included in the base
+    # library, and so is set as the default. The way this rollout works is by
+    # generating a crc from the experiment run key, which ensures that a given
+    # context will always be assigned the same variant.
+    #
+    # Distribution rules can be specified using an array or a hash, and if no
+    # rules are provided the default is to assign even distribution across all
+    # variants.
+    #
+    #   class MyExperiment < ActiveExperiment::Base
+    #     control { }
+    #     variant(:red) { }
+    #     variant(:blue) { }
+    #
+    #     # Assign even distribution to all variants.
+    #     rollout :percent
+    #
+    #     # Assign 25% to control, 30% to red, and 45% to blue.
+    #     rollout :percent, rules: {control: 25, red: 30, blue: 45}
+    #
+    #     # Same as above, but using an array.
+    #     rollout :percent, rules: [25, 30, 45]
+    #   end
+    #
+    # To use as the default, configure it to +:percent+.
+    #
+    #   ActiveExperiment::Base.default_rollout = :percent
+    #   Rails.application.config.active_experiment.default_rollout = :percent
+    class PercentRollout < BaseRollout
+      def initialize(experiment_class, ...) # :nodoc:
+        super
+
+        validate!(experiment_class)
+      end
+
+      def variant_for(experiment) # :nodoc:
+        variants = experiment.variant_names
+        crc = Zlib.crc32(experiment.run_key, 0)
+        total = 0
+
+        case rules
+        when Array then variants[rules.find_index { |percent| crc % 100 <= total += percent }]
+        when Hash then rules.find { |_, percent| crc % 100 <= total += percent }.first
+        else variants[crc % variants.length]
+        end
+      end
+
+      private
+        def validate!(experiment_class)
+          variant_names = experiment_class.try(:variants)&.keys
+          return if variant_names.blank?
+
+          case rules
+          when Hash
+            sum = rules.values.sum
+            raise ArgumentError, "The provided rules total #{sum}%, but should be 100%" if sum != 100
+
+            diff = rules.keys - variant_names | variant_names - rules.keys
+            raise ArgumentError, "The provided rules don't match the variants: #{diff.join(", ")}" if diff.any?
+          when Array
+            sum = rules.sum
+            raise ArgumentError, "The provided rules total #{sum}%, but should be 100%" if sum != 100
+
+            diff = rules.length - variant_names.length
+            raise ArgumentError, "The provided rules don't match the number of variants" if diff != 0
+          else
+            raise ArgumentError unless rules.nil?
+          end
+        end
+
+        def rules
+          @rollout_options[:rules]
+        end
+    end
+  end
+end

data/lib/active_experiment/rollouts/random_rollout.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  module Rollouts
+    # == Active Experiment Random Rollout
+    #
+    # The random rollout will assign a random variant every time the experiment
+    # is run.
+    #
+    # The behavior with random assignment is dependent on if caching is being
+    # used or not. This can be specified by providing the +cache:+ option to
+    # the rollout.
+    #
+    # When caching, the same variant will be assigned given the same experiment
+    # context, and will also slowly increases the number of contexts included
+    # in the experiment since with each run there's a chance that a context can
+    # be promoted out of the control group.
+    #
+    #   class MyExperiment < ActiveExperiment::Base
+    #     control { }
+    #     variant(:red) { }
+    #     variant(:blue) { }
+    #
+    #     # Randomize between all variants, every run.
+    #     rollout :random
+    #
+    #     # Random, but once assigned, cache the assignment.
+    #     rollout :random, cache: true
+    #   end
+    #
+    # To use as the default, configure it to +:random+.
+    #
+    #   ActiveExperiment::Base.default_rollout = :random
+    #   Rails.application.config.active_experiment.default_rollout = :random
+    class RandomRollout < BaseRollout
+      def variant_for(experiment) # :nodoc:
+        if @rollout_options[:cache]
+          experiment.variant_names.sample
+        else
+          experiment.set(variant: experiment.variant_names.sample)
+          nil # returning nil bypasses caching
+        end
+      end
+    end
+  end
+end

data/lib/active_experiment/rollouts.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Included Rollouts
+  #
+  # Active Experiment provides a few base rollout concepts that can be used to
+  # determine if an experiment should be skipped, and which variant to assign.
+  #
+  # A default rollout can be configured globally, and different rollouts can be
+  # specified on a per-experiment basis. Rollouts aren't inherited from parent
+  # classes.
+  #
+  # The included rollouts are:
+  #
+  # * +:random+ - Randomly assigns a variant (each run, or once with caching).
+  # * +:percent+ - Assigns a variant based on distribution rules, or evenly.
+  #
+  # == Custom Rollouts
+  #
+  # Custom rollouts can be created and registered with Active Experiment. A
+  # rollout must implement two methods to be considered valid, which can be
+  # achieved by inheriting the base class or one of the included rollouts.
+  #
+  # To illustrate, here's a simple rollout based on a fictional feature flag
+  # library that also assigns a random variant.
+  #
+  #   class FeatureFlagRollout < ActiveExperiment::Rollouts::BaseRollout
+  #     def skipped_for(experiment)
+  #       !FeatureFlag.enabled?(@rollout_options[:flag_name] || experiment.name)
+  #     end
+  #
+  #     def variant_for(experiment)
+  #       experiment.variant_names.sample
+  #     end
+  #   end
+  #
+  # This can now be registered and used the same way the included rollouts are:
+  #
+  #   ActiveExperiment::Rollouts.register(:feature_flag, FeatureFlagRollout)
+  #
+  # After registering the custom rollout, it can be used in experiments:
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { }
+  #     variant(:blue) { }
+  #
+  #     use_rollout :feature_flag, flag_name: "my_feature_flag"
+  #   end
+  #
+  # Or it can be configured as the default rollout for all experiments:
+  #
+  #   ActiveExperiment::Base.default_rollout = :feature_flag
+  #
+  # Custom rollouts can also be registered using autoloading. For example, if a
+  # custom rollout is defined in +lib/feature_flag_rollout.rb+, it can be
+  # registered to be autoloaded, and is only loaded when needed.
+  #
+  #   ActiveExperiment::Rollouts.register(
+  #     :feature_flag,
+  #     Rails.root.join("lib/feature_flag_rollout.rb")
+  #   )
+  #
+  # Now, the custom rollout will only be loaded when used in an experiment.
+  module Rollouts
+    extend ActiveSupport::Autoload
+
+    autoload :InactiveRollout
+    autoload :PercentRollout
+    autoload :RandomRollout
+
+    ROLLOUT_SUFFIX = "Rollout"
+    private_constant :ROLLOUT_SUFFIX
+
+    # Allows registering custom rollouts.
+    #
+    # The rollout must implement the +skipped_for+ and +variant_for+ methods,
+    # which is checked when the rollout is used in an experiment.
+    #
+    # If a string or +Pathname+ is provided, the rollout will be autoloaded.
+    #
+    # Raises an +ArgumentError+ if the rollout isn't an expected type.
+    def self.register(name, rollout)
+      const_name = "#{name.to_s.camelize}#{ROLLOUT_SUFFIX}"
+      case rollout
+      when String, Pathname
+        autoload(const_name, rollout)
+      when Class
+        const_set(const_name, rollout)
+      else
+        raise ArgumentError, "Provide a class to register, or string for autoloading"
+      end
+    end
+
+    # Allows looking up a rollout by name.
+    #
+    # Raises an +ArgumentError+ if the rollout hasn't been registered.
+    def self.lookup(name)
+      const_get("#{name.to_s.camelize}#{ROLLOUT_SUFFIX}")
+    rescue NameError
+      raise ArgumentError, "No rollout registered for #{name.inspect}"
+    end
+
+    # Base class for the included rollouts. Useful for custom rollouts.
+    #
+    # Any rollout that inherits from this class will be valid, not skipped, and
+    # will assign the first defined variant unless the provided methods are
+    # overridden.
+    class BaseRollout
+      def initialize(experiment_class, *args, **options, &block) # :nodoc:
+        @experiment_class = experiment_class
+        @rollout_args = args
+        @rollout_options = options
+        yield if block
+      end
+
+      # The base rollout is never skipped.
+      def skipped_for(_experiment)
+        false
+      end
+
+      # The base rollout always assigns the first variant.
+      def variant_for(experiment)
+        experiment.variant_names.first
+      end
+    end
+  end
+end

data/lib/active_experiment/rspec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  config.include ActiveExperiment::TestHelper, type: :experiment
+
+  config.before(:each, type: :experiment) { clear_executed_experiments }
+  config.after(:each, type: :experiment) { clear_executed_experiments }
+
+  config.define_derived_metadata(file_path: Regexp.new("spec/experiments/")) do |metadata|
+    metadata[:type] ||= :experiment
+  end
+end

data/lib/active_experiment/run_key.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "digest/sha2"
+
+module ActiveExperiment
+  # == Run Keys
+  #
+  # SHA2 is used to generate a hexdigest from an experiment context. This
+  # is generally referred to as the run key and can be used as the cache key
+  # and for variant assignment.
+  #
+  # You can configure the details used in generating the digest by specifying a
+  # secret key and a bit length. The secret key is used to salt the digest, and
+  # the bit length is used to determine the length of the digest.
+  #
+  # The secret key will default to +Rails.application.secrets.secret_key_base+
+  # when possible, and can be configured by:
+  #
+  #   ActiveExperiment::Base.digest_secret_key = ENV["AE_SECRET_KEY"]
+  #
+  # The bit length can be set to 256, 384, or 512. The default is 256, and this
+  # can be configured by:
+  #
+  #   ActiveExperiment::Base.digest_bit_length = 256
+  module RunKey
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :digest_secret_key, instance_writer: false, instance_predicate: false
+      class_attribute :digest_bit_length, instance_writer: false, instance_predicate: false, default: 256
+      private :digest_secret_key, :digest_bit_length
+    end
+
+    private
+      def run_key_hexdigest(source)
+        source = source.keys + source.values if source.is_a?(Hash)
+        ingredients = Array(source).map { |value| identify_object(value).inspect }
+        ingredients.unshift(name, digest_secret_key)
+
+        ::Digest::SHA2.new(digest_bit_length).hexdigest(ingredients.join("|"))
+      end
+
+      def identify_object(arg)
+        case arg
+        when GlobalID::Identification
+          arg.to_global_id.to_s rescue arg
+        else
+          # TODO: maybe we should strip things out that might cause issues?
+          #   e.g. `#<User:0x00007f9b0a0b0e60>` is going to change every run,
+          #   and we don't want that to happen by accident.
+          arg
+        end
+      end
+  end
+end

data/lib/active_experiment/segments.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module ActiveExperiment
+  # == Segmentation
+  #
+  # Segment rules are used to assign a specific variant in certain cases, and
+  # allows customized logic when resolving variants. Rules are evaluated in the
+  # order they're defined, and if a rule returns +true+, subsequent rules will
+  # be skipped.
+  #
+  # Segment rules are callbacks behind the scenes, so they accept the same set
+  # of options as other common Rails callbacks, including +if:+ and +unless:+
+  # which allows creating more complex rules.
+  #
+  # In the following example, any context with the name "Richard" will be
+  # assigned the red variant, and any context created more than 7 days ago will
+  # be assigned the blue variant.
+  #
+  #   class MyExperiment < ActiveExperiment::Base
+  #     variant(:red) { "red" }
+  #     variant(:blue) { "blue" }
+  #
+  #     segment :old_accounts, into: :red
+  #     segment(into: :blue) { context.name == "Richard" }
+  #     segment into: :red, if: :opted_in?
+  #
+  #     private
+  #
+  #     def old_accounts
+  #       context.created_at < 1.week.ago
+  #     end
+  #
+  #     def opted_in?
+  #       context.opted_in?
+  #     end
+  #   end
+  #
+  # This experiment now depends on something like a +User+ record being
+  # provided as context, since the context is now used to determine the variant
+  # through segment rules.
+  #
+  # Since rules are evaluated in the order they're defined, and the variant of
+  # the first rule to return true will be assigned. In the above example, this
+  # means that all old accounts will be put into the red variant regardless of
+  # being named Richard, and Richard can never "opt in" for the red variant.
+  module Segments
+    extend ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    included do
+      define_callbacks :segment
+      private :_segment_callbacks, :_run_segment_callbacks
+    end
+
+    # These methods will be included into any Active Experiment object, adding
+    # the segment method.
+    module ClassMethods
+      private
+        def segment(*filters, into:, **options, &block)
+          raise ArgumentError, "Unknown #{into} variant" unless variants[into.to_sym]
+
+          filters = filters.unshift(block)
+          set_callback_with_target(:segment, *filters, default: -> { true }, **options) do |target, callback|
+            target.set(variant: into) && throw(:abort) if true == callback.call(target, nil)
+          end
+        end
+    end
+  end
+end

data/lib/active_experiment/test_case.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "active_support/test_case"
+
+module ActiveExperiment
+  class TestCase < ActiveSupport::TestCase
+    include ActiveExperiment::TestHelper
+
+    ActiveSupport.run_load_hooks(:active_experiment_test_case, self)
+  end
+end