activeexperiment 0.1.0.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +0 -0
  3. data/MIT-LICENSE +20 -0
  4. data/README.md +373 -0
  5. data/lib/active_experiment/base.rb +76 -0
  6. data/lib/active_experiment/cache/active_record_cache_store.rb +116 -0
  7. data/lib/active_experiment/cache/redis_hash_cache_store.rb +68 -0
  8. data/lib/active_experiment/cache.rb +28 -0
  9. data/lib/active_experiment/caching.rb +172 -0
  10. data/lib/active_experiment/callbacks.rb +111 -0
  11. data/lib/active_experiment/capturable.rb +117 -0
  12. data/lib/active_experiment/configured_experiment.rb +74 -0
  13. data/lib/active_experiment/core.rb +177 -0
  14. data/lib/active_experiment/executed.rb +86 -0
  15. data/lib/active_experiment/execution.rb +156 -0
  16. data/lib/active_experiment/gem_version.rb +18 -0
  17. data/lib/active_experiment/instrumentation.rb +45 -0
  18. data/lib/active_experiment/log_subscriber.rb +178 -0
  19. data/lib/active_experiment/logging.rb +62 -0
  20. data/lib/active_experiment/railtie.rb +69 -0
  21. data/lib/active_experiment/rollout.rb +106 -0
  22. data/lib/active_experiment/rollouts/inactive_rollout.rb +24 -0
  23. data/lib/active_experiment/rollouts/percent_rollout.rb +84 -0
  24. data/lib/active_experiment/rollouts/random_rollout.rb +46 -0
  25. data/lib/active_experiment/rollouts.rb +127 -0
  26. data/lib/active_experiment/rspec.rb +12 -0
  27. data/lib/active_experiment/run_key.rb +55 -0
  28. data/lib/active_experiment/segments.rb +69 -0
  29. data/lib/active_experiment/test_case.rb +11 -0
  30. data/lib/active_experiment/test_helper.rb +267 -0
  31. data/lib/active_experiment/variants.rb +145 -0
  32. data/lib/active_experiment/version.rb +11 -0
  33. data/lib/active_experiment.rb +27 -0
  34. data/lib/activeexperiment.rb +8 -0
  35. data/lib/rails/generators/experiment/USAGE +12 -0
  36. data/lib/rails/generators/experiment/experiment_generator.rb +53 -0
  37. data/lib/rails/generators/experiment/templates/application_experiment.rb.tt +4 -0
  38. data/lib/rails/generators/experiment/templates/experiment.rb.tt +35 -0
  39. data/lib/rails/generators/rspec/experiment/experiment_generator.rb +20 -0
  40. data/lib/rails/generators/rspec/experiment/templates/experiment_spec.rb.tt +7 -0
  41. data/lib/rails/generators/test_unit/experiment/experiment_generator.rb +21 -0
  42. data/lib/rails/generators/test_unit/experiment/templates/experiment_test.rb.tt +9 -0
  43. metadata +118 -0
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Rollout Module
5
+ #
6
+ # Active Experiment can be configured to use one of the built in rollouts, or
7
+ # a custom rollout.
8
+ #
9
+ # When configuring the default rollout, you can use a symbol or something
10
+ # that responds to the required +skipped_for+ and +variant_for+ methods. To
11
+ # configure the default rollout:
12
+ #
13
+ # ActiveExperiment::Base.default_rollout = :random
14
+ #
15
+ # The above example will use the built in +:random+ rollout for all
16
+ # experiments. A given experiment can also be configured to use a specific
17
+ # rollout which will override the default:
18
+ #
19
+ # class MyExperiment < ActiveExperiment::Base
20
+ # variant(:red) { "red" }
21
+ # variant(:blue) { "blue" }
22
+ #
23
+ # rollout :percent, rules: { blue: 60, red: 40 }
24
+ # end
25
+ #
26
+ # An experiment might even be configured to use itself as a rollout. As
27
+ # long as the class responds to the required methods, it can be used.
28
+ #
29
+ # module ExperimentRolloutExample
30
+ # def skipped_for(*)
31
+ # false
32
+ # end
33
+ #
34
+ # def variant_for(*)
35
+ # :red
36
+ # end
37
+ # end
38
+ #
39
+ # class MyExperiment < ActiveExperiment::Base
40
+ # extend ExperimentRolloutExample
41
+ #
42
+ # variant(:red) { "red" }
43
+ # variant(:blue) { "blue" }
44
+ #
45
+ # use_rollout self
46
+ # end
47
+ #
48
+ # The flexibility that this affords is that you can use any rollout you want
49
+ # for any experiment. Rollouts can be defined that use a database, feature
50
+ # flags, or any other mechanism you want.
51
+ module Rollout
52
+ extend ActiveSupport::Concern
53
+
54
+ REQUIRED_ROLLOUT_METHODS = [:skipped_for, :variant_for].freeze
55
+ private_constant :REQUIRED_ROLLOUT_METHODS
56
+
57
+ included do
58
+ class_attribute :rollout, instance_predicate: false
59
+ private :rollout=
60
+
61
+ self.default_rollout = :percent
62
+ end
63
+
64
+ # These methods will be included into any Active Experiment object and
65
+ # allow setting the default rollout, the experiment specific rollout, and
66
+ # includes the inherited behavior for default rollouts. When setting the
67
+ # rollout, it will be validated to ensure it responds to the required
68
+ # methods.
69
+ module ClassMethods
70
+ def inherited(subclass) # :nodoc:
71
+ super
72
+ subclass.default_rollout = @default_rollout
73
+ end
74
+
75
+ # Allows setting the default rollout for all experiments.
76
+ #
77
+ # This can be overridden on a per experiment basis, but overrides to the
78
+ # default rollout are not be inherited. Meaning that each experiment will
79
+ # revert to the default rollout, regardless of what it inherits from.
80
+ def default_rollout=(name_or_rollout)
81
+ use_rollout(name_or_rollout)
82
+ @default_rollout = name_or_rollout
83
+ end
84
+
85
+ private
86
+ def use_rollout(name_or_rollout, *args, **kws, &block)
87
+ case name_or_rollout
88
+ when Symbol, String
89
+ rollout = ActiveExperiment::Rollouts.lookup(name_or_rollout)
90
+ self.rollout = rollout.new(self, *args, **kws, &block)
91
+ else
92
+ unless rollout_interface?(name_or_rollout)
93
+ raise ArgumentError, "Invalid rollout. " \
94
+ "Rollouts must respond to #{REQUIRED_ROLLOUT_METHODS.join(", ")}."
95
+ end
96
+
97
+ self.rollout = name_or_rollout
98
+ end
99
+ end
100
+
101
+ def rollout_interface?(object)
102
+ REQUIRED_ROLLOUT_METHODS.all? { |meth| object.respond_to?(meth) }
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ module Rollouts
5
+ # == Active Experiment Inactive Rollout
6
+ #
7
+ # Using this rollout will disable experiments as though they were
8
+ # intentionally skipped.
9
+ #
10
+ # To use as the default, configure it to +:inactive+.
11
+ #
12
+ # ActiveExperiment::Base.default_rollout = :inactive
13
+ # Rails.application.config.active_experiment.default_rollout = :inactive
14
+ class InactiveRollout < BaseRollout
15
+ def skipped_for(*)
16
+ true
17
+ end
18
+
19
+ def variant_for(*)
20
+ nil
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zlib"
4
+
5
+ module ActiveExperiment
6
+ module Rollouts
7
+ # == Active Experiment Percent Rollout
8
+ #
9
+ # The percent rollout is the most comprehensive included in the base
10
+ # library, and so is set as the default. The way this rollout works is by
11
+ # generating a crc from the experiment run key, which ensures that a given
12
+ # context will always be assigned the same variant.
13
+ #
14
+ # Distribution rules can be specified using an array or a hash, and if no
15
+ # rules are provided the default is to assign even distribution across all
16
+ # variants.
17
+ #
18
+ # class MyExperiment < ActiveExperiment::Base
19
+ # control { }
20
+ # variant(:red) { }
21
+ # variant(:blue) { }
22
+ #
23
+ # # Assign even distribution to all variants.
24
+ # rollout :percent
25
+ #
26
+ # # Assign 25% to control, 30% to red, and 45% to blue.
27
+ # rollout :percent, rules: {control: 25, red: 30, blue: 45}
28
+ #
29
+ # # Same as above, but using an array.
30
+ # rollout :percent, rules: [25, 30, 45]
31
+ # end
32
+ #
33
+ # To use as the default, configure it to +:percent+.
34
+ #
35
+ # ActiveExperiment::Base.default_rollout = :percent
36
+ # Rails.application.config.active_experiment.default_rollout = :percent
37
+ class PercentRollout < BaseRollout
38
+ def initialize(experiment_class, ...) # :nodoc:
39
+ super
40
+
41
+ validate!(experiment_class)
42
+ end
43
+
44
+ def variant_for(experiment) # :nodoc:
45
+ variants = experiment.variant_names
46
+ crc = Zlib.crc32(experiment.run_key, 0)
47
+ total = 0
48
+
49
+ case rules
50
+ when Array then variants[rules.find_index { |percent| crc % 100 <= total += percent }]
51
+ when Hash then rules.find { |_, percent| crc % 100 <= total += percent }.first
52
+ else variants[crc % variants.length]
53
+ end
54
+ end
55
+
56
+ private
57
+ def validate!(experiment_class)
58
+ variant_names = experiment_class.try(:variants)&.keys
59
+ return if variant_names.blank?
60
+
61
+ case rules
62
+ when Hash
63
+ sum = rules.values.sum
64
+ raise ArgumentError, "The provided rules total #{sum}%, but should be 100%" if sum != 100
65
+
66
+ diff = rules.keys - variant_names | variant_names - rules.keys
67
+ raise ArgumentError, "The provided rules don't match the variants: #{diff.join(", ")}" if diff.any?
68
+ when Array
69
+ sum = rules.sum
70
+ raise ArgumentError, "The provided rules total #{sum}%, but should be 100%" if sum != 100
71
+
72
+ diff = rules.length - variant_names.length
73
+ raise ArgumentError, "The provided rules don't match the number of variants" if diff != 0
74
+ else
75
+ raise ArgumentError unless rules.nil?
76
+ end
77
+ end
78
+
79
+ def rules
80
+ @rollout_options[:rules]
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ module Rollouts
5
+ # == Active Experiment Random Rollout
6
+ #
7
+ # The random rollout will assign a random variant every time the experiment
8
+ # is run.
9
+ #
10
+ # The behavior with random assignment is dependent on if caching is being
11
+ # used or not. This can be specified by providing the +cache:+ option to
12
+ # the rollout.
13
+ #
14
+ # When caching, the same variant will be assigned given the same experiment
15
+ # context, and will also slowly increases the number of contexts included
16
+ # in the experiment since with each run there's a chance that a context can
17
+ # be promoted out of the control group.
18
+ #
19
+ # class MyExperiment < ActiveExperiment::Base
20
+ # control { }
21
+ # variant(:red) { }
22
+ # variant(:blue) { }
23
+ #
24
+ # # Randomize between all variants, every run.
25
+ # rollout :random
26
+ #
27
+ # # Random, but once assigned, cache the assignment.
28
+ # rollout :random, cache: true
29
+ # end
30
+ #
31
+ # To use as the default, configure it to +:random+.
32
+ #
33
+ # ActiveExperiment::Base.default_rollout = :random
34
+ # Rails.application.config.active_experiment.default_rollout = :random
35
+ class RandomRollout < BaseRollout
36
+ def variant_for(experiment) # :nodoc:
37
+ if @rollout_options[:cache]
38
+ experiment.variant_names.sample
39
+ else
40
+ experiment.set(variant: experiment.variant_names.sample)
41
+ nil # returning nil bypasses caching
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Included Rollouts
5
+ #
6
+ # Active Experiment provides a few base rollout concepts that can be used to
7
+ # determine if an experiment should be skipped, and which variant to assign.
8
+ #
9
+ # A default rollout can be configured globally, and different rollouts can be
10
+ # specified on a per-experiment basis. Rollouts aren't inherited from parent
11
+ # classes.
12
+ #
13
+ # The included rollouts are:
14
+ #
15
+ # * +:random+ - Randomly assigns a variant (each run, or once with caching).
16
+ # * +:percent+ - Assigns a variant based on distribution rules, or evenly.
17
+ #
18
+ # == Custom Rollouts
19
+ #
20
+ # Custom rollouts can be created and registered with Active Experiment. A
21
+ # rollout must implement two methods to be considered valid, which can be
22
+ # achieved by inheriting the base class or one of the included rollouts.
23
+ #
24
+ # To illustrate, here's a simple rollout based on a fictional feature flag
25
+ # library that also assigns a random variant.
26
+ #
27
+ # class FeatureFlagRollout < ActiveExperiment::Rollouts::BaseRollout
28
+ # def skipped_for(experiment)
29
+ # !FeatureFlag.enabled?(@rollout_options[:flag_name] || experiment.name)
30
+ # end
31
+ #
32
+ # def variant_for(experiment)
33
+ # experiment.variant_names.sample
34
+ # end
35
+ # end
36
+ #
37
+ # This can now be registered and used the same way the included rollouts are:
38
+ #
39
+ # ActiveExperiment::Rollouts.register(:feature_flag, FeatureFlagRollout)
40
+ #
41
+ # After registering the custom rollout, it can be used in experiments:
42
+ #
43
+ # class MyExperiment < ActiveExperiment::Base
44
+ # variant(:red) { }
45
+ # variant(:blue) { }
46
+ #
47
+ # use_rollout :feature_flag, flag_name: "my_feature_flag"
48
+ # end
49
+ #
50
+ # Or it can be configured as the default rollout for all experiments:
51
+ #
52
+ # ActiveExperiment::Base.default_rollout = :feature_flag
53
+ #
54
+ # Custom rollouts can also be registered using autoloading. For example, if a
55
+ # custom rollout is defined in +lib/feature_flag_rollout.rb+, it can be
56
+ # registered to be autoloaded, and is only loaded when needed.
57
+ #
58
+ # ActiveExperiment::Rollouts.register(
59
+ # :feature_flag,
60
+ # Rails.root.join("lib/feature_flag_rollout.rb")
61
+ # )
62
+ #
63
+ # Now, the custom rollout will only be loaded when used in an experiment.
64
+ module Rollouts
65
+ extend ActiveSupport::Autoload
66
+
67
+ autoload :InactiveRollout
68
+ autoload :PercentRollout
69
+ autoload :RandomRollout
70
+
71
+ ROLLOUT_SUFFIX = "Rollout"
72
+ private_constant :ROLLOUT_SUFFIX
73
+
74
+ # Allows registering custom rollouts.
75
+ #
76
+ # The rollout must implement the +skipped_for+ and +variant_for+ methods,
77
+ # which is checked when the rollout is used in an experiment.
78
+ #
79
+ # If a string or +Pathname+ is provided, the rollout will be autoloaded.
80
+ #
81
+ # Raises an +ArgumentError+ if the rollout isn't an expected type.
82
+ def self.register(name, rollout)
83
+ const_name = "#{name.to_s.camelize}#{ROLLOUT_SUFFIX}"
84
+ case rollout
85
+ when String, Pathname
86
+ autoload(const_name, rollout)
87
+ when Class
88
+ const_set(const_name, rollout)
89
+ else
90
+ raise ArgumentError, "Provide a class to register, or string for autoloading"
91
+ end
92
+ end
93
+
94
+ # Allows looking up a rollout by name.
95
+ #
96
+ # Raises an +ArgumentError+ if the rollout hasn't been registered.
97
+ def self.lookup(name)
98
+ const_get("#{name.to_s.camelize}#{ROLLOUT_SUFFIX}")
99
+ rescue NameError
100
+ raise ArgumentError, "No rollout registered for #{name.inspect}"
101
+ end
102
+
103
+ # Base class for the included rollouts. Useful for custom rollouts.
104
+ #
105
+ # Any rollout that inherits from this class will be valid, not skipped, and
106
+ # will assign the first defined variant unless the provided methods are
107
+ # overridden.
108
+ class BaseRollout
109
+ def initialize(experiment_class, *args, **options, &block) # :nodoc:
110
+ @experiment_class = experiment_class
111
+ @rollout_args = args
112
+ @rollout_options = options
113
+ yield if block
114
+ end
115
+
116
+ # The base rollout is never skipped.
117
+ def skipped_for(_experiment)
118
+ false
119
+ end
120
+
121
+ # The base rollout always assigns the first variant.
122
+ def variant_for(experiment)
123
+ experiment.variant_names.first
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ RSpec.configure do |config|
4
+ config.include ActiveExperiment::TestHelper, type: :experiment
5
+
6
+ config.before(:each, type: :experiment) { clear_executed_experiments }
7
+ config.after(:each, type: :experiment) { clear_executed_experiments }
8
+
9
+ config.define_derived_metadata(file_path: Regexp.new("spec/experiments/")) do |metadata|
10
+ metadata[:type] ||= :experiment
11
+ end
12
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest/sha2"
4
+
5
+ module ActiveExperiment
6
+ # == Run Keys
7
+ #
8
+ # SHA2 is used to generate a hexdigest from an experiment context. This
9
+ # is generally referred to as the run key and can be used as the cache key
10
+ # and for variant assignment.
11
+ #
12
+ # You can configure the details used in generating the digest by specifying a
13
+ # secret key and a bit length. The secret key is used to salt the digest, and
14
+ # the bit length is used to determine the length of the digest.
15
+ #
16
+ # The secret key will default to +Rails.application.secrets.secret_key_base+
17
+ # when possible, and can be configured by:
18
+ #
19
+ # ActiveExperiment::Base.digest_secret_key = ENV["AE_SECRET_KEY"]
20
+ #
21
+ # The bit length can be set to 256, 384, or 512. The default is 256, and this
22
+ # can be configured by:
23
+ #
24
+ # ActiveExperiment::Base.digest_bit_length = 256
25
+ module RunKey
26
+ extend ActiveSupport::Concern
27
+
28
+ included do
29
+ class_attribute :digest_secret_key, instance_writer: false, instance_predicate: false
30
+ class_attribute :digest_bit_length, instance_writer: false, instance_predicate: false, default: 256
31
+ private :digest_secret_key, :digest_bit_length
32
+ end
33
+
34
+ private
35
+ def run_key_hexdigest(source)
36
+ source = source.keys + source.values if source.is_a?(Hash)
37
+ ingredients = Array(source).map { |value| identify_object(value).inspect }
38
+ ingredients.unshift(name, digest_secret_key)
39
+
40
+ ::Digest::SHA2.new(digest_bit_length).hexdigest(ingredients.join("|"))
41
+ end
42
+
43
+ def identify_object(arg)
44
+ case arg
45
+ when GlobalID::Identification
46
+ arg.to_global_id.to_s rescue arg
47
+ else
48
+ # TODO: maybe we should strip things out that might cause issues?
49
+ # e.g. `#<User:0x00007f9b0a0b0e60>` is going to change every run,
50
+ # and we don't want that to happen by accident.
51
+ arg
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Segmentation
5
+ #
6
+ # Segment rules are used to assign a specific variant in certain cases, and
7
+ # allows customized logic when resolving variants. Rules are evaluated in the
8
+ # order they're defined, and if a rule returns +true+, subsequent rules will
9
+ # be skipped.
10
+ #
11
+ # Segment rules are callbacks behind the scenes, so they accept the same set
12
+ # of options as other common Rails callbacks, including +if:+ and +unless:+
13
+ # which allows creating more complex rules.
14
+ #
15
+ # In the following example, any context with the name "Richard" will be
16
+ # assigned the red variant, and any context created more than 7 days ago will
17
+ # be assigned the blue variant.
18
+ #
19
+ # class MyExperiment < ActiveExperiment::Base
20
+ # variant(:red) { "red" }
21
+ # variant(:blue) { "blue" }
22
+ #
23
+ # segment :old_accounts, into: :red
24
+ # segment(into: :blue) { context.name == "Richard" }
25
+ # segment into: :red, if: :opted_in?
26
+ #
27
+ # private
28
+ #
29
+ # def old_accounts
30
+ # context.created_at < 1.week.ago
31
+ # end
32
+ #
33
+ # def opted_in?
34
+ # context.opted_in?
35
+ # end
36
+ # end
37
+ #
38
+ # This experiment now depends on something like a +User+ record being
39
+ # provided as context, since the context is now used to determine the variant
40
+ # through segment rules.
41
+ #
42
+ # Since rules are evaluated in the order they're defined, and the variant of
43
+ # the first rule to return true will be assigned. In the above example, this
44
+ # means that all old accounts will be put into the red variant regardless of
45
+ # being named Richard, and Richard can never "opt in" for the red variant.
46
+ module Segments
47
+ extend ActiveSupport::Concern
48
+ include ActiveSupport::Callbacks
49
+
50
+ included do
51
+ define_callbacks :segment
52
+ private :_segment_callbacks, :_run_segment_callbacks
53
+ end
54
+
55
+ # These methods will be included into any Active Experiment object, adding
56
+ # the segment method.
57
+ module ClassMethods
58
+ private
59
+ def segment(*filters, into:, **options, &block)
60
+ raise ArgumentError, "Unknown #{into} variant" unless variants[into.to_sym]
61
+
62
+ filters = filters.unshift(block)
63
+ set_callback_with_target(:segment, *filters, default: -> { true }, **options) do |target, callback|
64
+ target.set(variant: into) && throw(:abort) if true == callback.call(target, nil)
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/test_case"
4
+
5
+ module ActiveExperiment
6
+ class TestCase < ActiveSupport::TestCase
7
+ include ActiveExperiment::TestHelper
8
+
9
+ ActiveSupport.run_load_hooks(:active_experiment_test_case, self)
10
+ end
11
+ end