activeexperiment 0.1.0.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +0 -0
  3. data/MIT-LICENSE +20 -0
  4. data/README.md +373 -0
  5. data/lib/active_experiment/base.rb +76 -0
  6. data/lib/active_experiment/cache/active_record_cache_store.rb +116 -0
  7. data/lib/active_experiment/cache/redis_hash_cache_store.rb +68 -0
  8. data/lib/active_experiment/cache.rb +28 -0
  9. data/lib/active_experiment/caching.rb +172 -0
  10. data/lib/active_experiment/callbacks.rb +111 -0
  11. data/lib/active_experiment/capturable.rb +117 -0
  12. data/lib/active_experiment/configured_experiment.rb +74 -0
  13. data/lib/active_experiment/core.rb +177 -0
  14. data/lib/active_experiment/executed.rb +86 -0
  15. data/lib/active_experiment/execution.rb +156 -0
  16. data/lib/active_experiment/gem_version.rb +18 -0
  17. data/lib/active_experiment/instrumentation.rb +45 -0
  18. data/lib/active_experiment/log_subscriber.rb +178 -0
  19. data/lib/active_experiment/logging.rb +62 -0
  20. data/lib/active_experiment/railtie.rb +69 -0
  21. data/lib/active_experiment/rollout.rb +106 -0
  22. data/lib/active_experiment/rollouts/inactive_rollout.rb +24 -0
  23. data/lib/active_experiment/rollouts/percent_rollout.rb +84 -0
  24. data/lib/active_experiment/rollouts/random_rollout.rb +46 -0
  25. data/lib/active_experiment/rollouts.rb +127 -0
  26. data/lib/active_experiment/rspec.rb +12 -0
  27. data/lib/active_experiment/run_key.rb +55 -0
  28. data/lib/active_experiment/segments.rb +69 -0
  29. data/lib/active_experiment/test_case.rb +11 -0
  30. data/lib/active_experiment/test_helper.rb +267 -0
  31. data/lib/active_experiment/variants.rb +145 -0
  32. data/lib/active_experiment/version.rb +11 -0
  33. data/lib/active_experiment.rb +27 -0
  34. data/lib/activeexperiment.rb +8 -0
  35. data/lib/rails/generators/experiment/USAGE +12 -0
  36. data/lib/rails/generators/experiment/experiment_generator.rb +53 -0
  37. data/lib/rails/generators/experiment/templates/application_experiment.rb.tt +4 -0
  38. data/lib/rails/generators/experiment/templates/experiment.rb.tt +35 -0
  39. data/lib/rails/generators/rspec/experiment/experiment_generator.rb +20 -0
  40. data/lib/rails/generators/rspec/experiment/templates/experiment_spec.rb.tt +7 -0
  41. data/lib/rails/generators/test_unit/experiment/experiment_generator.rb +21 -0
  42. data/lib/rails/generators/test_unit/experiment/templates/experiment_test.rb.tt +9 -0
  43. metadata +118 -0
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/current_attributes"
4
+
5
+ module ActiveExperiment
6
+ # == Executed Experiments
7
+ #
8
+ # This is useful for surfacing experiments to the client layer. For example,
9
+ # if you have a client only experiment, an empty experiment can be defined
10
+ # and then run in the controller layer. The client layer can then use the
11
+ # experiment information to render the appropriate client code based on the
12
+ # variant that's been assigned, or other information available the experiment
13
+ # can provide in its +serialize+ method.
14
+ #
15
+ # An example of an empty experiment might be as simple as:
16
+ #
17
+ # class MyClientExperiment < ActiveExperiment::Base
18
+ # variant(:red) { }
19
+ # variant(:blue) { }
20
+ # end
21
+ #
22
+ # In the controller, this experiment can be run in a before action, or
23
+ # anywhere else that makes sense for your application:
24
+ #
25
+ # class MyController < ApplicationController
26
+ # before_action :run_my_client_experiment
27
+ # # ... controller code
28
+ #
29
+ # private
30
+ #
31
+ # def run_my_client_experiment
32
+ # MyClientExperiment.run(current_user)
33
+ # end
34
+ # end
35
+ #
36
+ # Then in the layout, or appropriate view, all experiments that have been run
37
+ # during the request can be surfaced:
38
+ #
39
+ # <title>My App</title>
40
+ # <script>
41
+ # window.experiments = <%= ActiveExperiment::Executed.to_json %>
42
+ # </script>
43
+ #
44
+ # Or the experiments that have been run can be iterated:
45
+ #
46
+ # <% ActiveExperiment::Executed.experiments.each do |experiment| %>
47
+ # <meta name="<%= experiment.name %>" content="<%= experiment.variant %>">
48
+ # <% end %>
49
+ class Executed < ActiveSupport::CurrentAttributes
50
+ attribute :experiments
51
+
52
+ # Interface to add an experiment to the executed experiments. This is
53
+ # intended to be used by the +run+ method of the experiment class.
54
+ #
55
+ # Experiments are added to the executed experiments if they have been
56
+ # assigned a variant.
57
+ def self.<<(experiment)
58
+ self.experiments ||= []
59
+ experiments << experiment
60
+ end
61
+
62
+ # Returns an array of experiments that have been run.
63
+ #
64
+ # Used by several other methods to return serialized experiments.
65
+ def self.as_array
66
+ self.experiments || []
67
+ end
68
+
69
+ # Returns a json of the experiments that have been run, with the experiment
70
+ # name as the key, and the serialized experiment as the value. This assumes
71
+ # that if an experiment is run multiple times, the same variant has been
72
+ # assigned for all runs, which may not be true, like when using the random
73
+ # rollout.
74
+ #
75
+ # When needed, the executed experiments can be accessed and/or iterated
76
+ # directly, or the +to_json_array+ method can be used.
77
+ def self.to_json
78
+ as_array.each_with_object({}) { |e, hash| hash[e.name] = e.serialize }.to_json
79
+ end
80
+
81
+ # Returns an array of experiments that have been run.
82
+ def self.to_json_array
83
+ as_array.map(&:serialize).to_json
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,156 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Execution Module
5
+ #
6
+ # This module provides most of the logic for running experiments. Running an
7
+ # experiment can be performed in a few ways, some of which are provided as
8
+ # convenience.
9
+ #
10
+ # 1. Calling +run+ on the class, passing the context and a block:
11
+ #
12
+ # MyExperiment.run(context) do |experiment|
13
+ # experiment.on(:treatment) { "treatment" }
14
+ # end
15
+ #
16
+ # 2. Instantiating the experiment with the context, and calling +run+:
17
+ #
18
+ # MyExperiment.new(context).run do |experiment|
19
+ # experiment.on(:treatment) { "treatment" }
20
+ # end
21
+ #
22
+ # 3. Using the +ConfiguredExperiment+ API to +set+ and then +run+:
23
+ #
24
+ # MyExperiment.set(variant: :treatment).run(id: 1) do |experiment|
25
+ # experiment.on(:treatment) { "treatment" }
26
+ # end
27
+ #
28
+ # In all cases, a block can be provided to the +run+ method. The block will
29
+ # be called with the experiment, which allows overriding the variant
30
+ # behaviors using the scope of where the experiment is being run.
31
+ #
32
+ # When the experiment is run, the variant will be determined and the variant
33
+ # steps will be executed. The result of the variant execution will be
34
+ # returned unless the experiment is aborted in a +before_run+ or
35
+ # +before_variant+ callback.
36
+ #
37
+ # In general, the following decision tree diagram helps illustrate the order
38
+ # that things will be executed in running an experiment, utilizing caching
39
+ # when possible:
40
+ # run
41
+ # |
42
+ # _ skipped? _
43
+ # | |
44
+ # no yes
45
+ # | |
46
+ # | assigned/default_variant
47
+ # |
48
+ # _ cached_variant? _
49
+ # | |
50
+ # no yes
51
+ # | |
52
+ # _ segmented? _ (cached value)
53
+ # | |
54
+ # yes no
55
+ # | |
56
+ # | ___ rollout.variant_for __
57
+ # | | | |
58
+ # (cache) (cache) (cache)
59
+ # variant_a variant_b variant_c
60
+ #
61
+ module Execution
62
+ extend ActiveSupport::Concern
63
+
64
+ # These methods will be included into any Active Experiment object and
65
+ # expose the class level run method, and the ability to get a configured
66
+ # experiment instance using the set method.
67
+ module ClassMethods
68
+ # Instantiates and runs an experiment with the provided context and
69
+ # block. This is a convenience method.
70
+ #
71
+ # An example of using this method to run an experiment:
72
+ #
73
+ # MyExperiment.run(id: 1) do |experiment|
74
+ # experiment.on(:treatment) { "red" }
75
+ # end
76
+ def run(*args, **kws, &block)
77
+ new(*args, **kws).run(&block)
78
+ end
79
+
80
+ # Creates a configured experiment with the provided options. Configured
81
+ # experiments expose a few helpful methods for running and caching
82
+ # experiment details.
83
+ #
84
+ # The following options can be provided to configure an experiment:
85
+ #
86
+ # * +:variant+ - The variant to assign.
87
+ #
88
+ # An example of using this method to set a variant and run an experiment:
89
+ #
90
+ # MyExperiment.set(variant: :red).run(id: 1) do |experiment|
91
+ # experiment.on(:red) { "red" }
92
+ # end
93
+ def set(**options)
94
+ ConfiguredExperiment.new(self, **options)
95
+ end
96
+ end
97
+
98
+ # Runs the experiment. Calling +run+ returns the value of the assigned
99
+ # variant block or method.
100
+ #
101
+ # When running an experiment, a block can be provided and it will be called
102
+ # with the experiment, which provides the ability to override variant
103
+ # behaviors when running the experiment.
104
+ #
105
+ # MyExperiment.new(id: 1).run do |experiment|
106
+ # experiment.on(:treatment) { "treatment" }
107
+ # end
108
+ #
109
+ # Raises an ActiveExperiment::ExecutionError if there are no variants
110
+ # registered, or if the experiment is already running, in the case of
111
+ # accidentally calling run again within a run or variant block.
112
+ def run(&block)
113
+ return @results if defined?(@results)
114
+ raise ExecutionError, "No variants registered" if variant_names.empty?
115
+
116
+ @results = nil
117
+ instrument(:start_experiment)
118
+ instrument(:process_run) do
119
+ run_callbacks(:run, :process_run_callbacks) do
120
+ call_run_block(&block) if block.present?
121
+ @variant = resolve_variant
122
+ @results = resolve_results
123
+ end
124
+ end
125
+
126
+ @results
127
+ ensure
128
+ Executed << self
129
+ end
130
+
131
+ private
132
+ def resolve_variant
133
+ return variant || default_variant if skipped?
134
+
135
+ resolved = cached_variant(variant) do
136
+ run_callbacks(:segment, :process_segment_callbacks)
137
+ variant || rollout.variant_for(self)
138
+ end
139
+
140
+ variant || resolved || default_variant
141
+ end
142
+
143
+ def resolve_results
144
+ resolved = nil
145
+ run_callbacks(variants[variant], :process_variant_callbacks) do
146
+ resolved = variant_step_chains[variant]&.call
147
+ end
148
+
149
+ resolved || @results
150
+ end
151
+
152
+ def call_run_block(&block)
153
+ block.call(self)
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # Returns the currently loaded version of Active Experiment as a
5
+ # +Gem::Version+.
6
+ def self.gem_version
7
+ Gem::Version.new(VERSION::STRING)
8
+ end
9
+
10
+ module VERSION
11
+ MAJOR = 0
12
+ MINOR = 1
13
+ TINY = 0
14
+ PRE = "alpha"
15
+
16
+ STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")
17
+ end
18
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Instrumentation Module
5
+ #
6
+ # Instrumentation is provided through +ActiveSupport::Notifications+. The
7
+ # best example of how to utilize the instrumentation within Active Experiment
8
+ # is look at how +ActiveExperiment::LogSubscriber+ has been implemented.
9
+ module Instrumentation
10
+ extend ActiveSupport::Concern
11
+
12
+ private
13
+ def run_callbacks(kind, event_name, **payload, &block)
14
+ if kind.present? && __callbacks[kind.to_sym].any?
15
+ instrument(event_name, **payload) do
16
+ super(kind, &block)
17
+ end
18
+ else
19
+ yield if block_given?
20
+ end
21
+ end
22
+
23
+ def instrument(operation, **payload, &block)
24
+ operation = "#{operation}.active_experiment"
25
+ payload = payload.merge(experiment: self)
26
+ return ActiveSupport::Notifications.instrument(operation, payload) unless block.present?
27
+
28
+ ActiveSupport::Notifications.instrument(operation, payload) do |event_payload|
29
+ variant = @variant
30
+ results = block.call
31
+
32
+ event_payload[:variant] = @variant if variant != @variant
33
+ event_payload[:aborted] = @halted_callback if @halted_callback.present?
34
+ @halted_callback = nil if @halted_callback == :segment
35
+
36
+ results
37
+ end
38
+ end
39
+
40
+ def halted_callback_hook(filter, name)
41
+ super
42
+ @halted_callback = name
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_support/log_subscriber"
4
+
5
+ module ActiveExperiment
6
+ # == Log Subscriber
7
+ #
8
+ # TODO: finish documenting.
9
+ class LogSubscriber < ActiveSupport::LogSubscriber
10
+ def start_experiment(event)
11
+ warn_of_nested_experiment(event) if execution_stack.any?
12
+ experiment_logger(event) do |experiment|
13
+ execution_stack.push(experiment)
14
+
15
+ info = []
16
+ info << "Run ID: #{experiment.run_id}"
17
+ info << "Variant: #{experiment.variant}" if experiment.variant.present?
18
+
19
+ build_message(:info, "Running #{experiment.name} (#{info.join(", ")})", context: experiment.log_context?)
20
+ end
21
+ end
22
+
23
+ def process_run(event)
24
+ errored = event.payload[:exception_object]
25
+ aborted = !errored && event.payload[:aborted]
26
+
27
+ experiment_logger(event) do |experiment|
28
+ execution_stack.pop
29
+
30
+ if errored
31
+ build_message(:error, "Run failed: #{errored.class} (#{errored.message})")
32
+ elsif aborted
33
+ build_message(:info, "Run aborted in #{aborted} callbacks", details: true)
34
+ else
35
+ variant_name = experiment.variant
36
+ if experiment.variant_names.include?(variant_name)
37
+ build_message(:info, "Completed running #{experiment.variant} variant", details: true)
38
+ elsif variant_name.present?
39
+ build_message(:error, "Run errored: unknown `#{variant_name}` variant resolved", details: true)
40
+ else
41
+ build_message(:error, "Run errored: no variant resolved", details: true)
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ def process_segment_callbacks(event)
48
+ return if event.payload[:exception_object].present?
49
+
50
+ experiment_logger(event) do |experiment|
51
+ if event.payload[:aborted] == :segment
52
+ build_message(:info, "Segmented into the `#{experiment.variant}` variant", duration: true)
53
+ else
54
+ build_callback_message(event)
55
+ end
56
+ end
57
+ end
58
+
59
+ def process_run_callbacks(event)
60
+ experiment_logger(event) { build_callback_message(event) }
61
+ end
62
+
63
+ def process_variant_callbacks(event)
64
+ experiment_logger(event) { build_callback_message(event) }
65
+ end
66
+
67
+ def process_variant_steps(event)
68
+ experiment_logger(event) { build_callback_message(event) }
69
+ end
70
+
71
+ private
72
+ def warn_of_nested_experiment(event)
73
+ experiment_logger(event) do
74
+ build_message(:warn, "Nesting experiment in #{experiment_identifier(execution_stack.last)}")
75
+ end
76
+ end
77
+
78
+ def experiment_logger(event, &block)
79
+ return unless logger.present?
80
+
81
+ experiment = event.payload[:experiment]
82
+ result = block.call(experiment)
83
+
84
+ return unless result.present?
85
+
86
+ logger.send(result[:level]) do
87
+ log = +colorized_prefix(experiment)
88
+ log << colorized_message(result[:message], level: result[:level])
89
+ log << colorized_duration(event, parens: true) if result[:duration]
90
+ log << colorized_details(event) if result[:details]
91
+ log << colorized_context(experiment) if result[:context]
92
+ log
93
+ end
94
+ end
95
+
96
+ def build_message(level, message, **kws)
97
+ { level: level, message: message, **kws }
98
+ end
99
+
100
+ def build_callback_message(event)
101
+ return if event.payload[:exception_object].present?
102
+
103
+ variant = event.payload[:variant]
104
+ process = event.name.split(".").first.gsub("process_", "").tr("_", " ")
105
+
106
+ if variant.present? && process != "run callbacks"
107
+ build_message(:info, "Resolved `#{variant}` variant in #{process}", duration: true)
108
+ elsif process != "variant steps"
109
+ build_message(:debug, "Completed #{process}", duration: true)
110
+ end
111
+ end
112
+
113
+ def colorized_prefix(experiment)
114
+ color(" #{experiment_identifier(experiment)} ", GREEN)
115
+ end
116
+
117
+ def colorized_message(message, level: :info)
118
+ case level
119
+ when :error
120
+ color(message, RED, bold: true)
121
+ when :warn
122
+ color(message, YELLOW, bold: true)
123
+ else
124
+ message
125
+ end
126
+ end
127
+
128
+ def colorized_details(event)
129
+ " (Duration:#{colorized_duration(event, parens: false)} | Allocations: #{event.allocations})"
130
+ end
131
+
132
+ def colorized_duration(event, parens: true)
133
+ duration = event.duration.round(1)
134
+ if duration > 1000
135
+ ret = color("#{duration}ms", RED, bold: true)
136
+ elsif duration > 500
137
+ ret = color("#{duration}ms", YELLOW, bold: true)
138
+ else
139
+ ret = "#{duration}ms"
140
+ end
141
+
142
+ parens ? " (#{ret})" : " #{ret}"
143
+ end
144
+
145
+ def colorized_context(experiment)
146
+ return "" unless experiment.log_context?
147
+
148
+ " with context: #{format_context(experiment.context).inspect}"
149
+ end
150
+
151
+ def format_context(arg)
152
+ case arg
153
+ when Hash
154
+ arg.transform_values { |value| format_context(value) }
155
+ when Array
156
+ arg.map { |value| format_context(value) }
157
+ when GlobalID::Identification
158
+ arg.to_global_id.to_s rescue arg
159
+ else
160
+ arg
161
+ end
162
+ end
163
+
164
+ def experiment_identifier(experiment)
165
+ "#{experiment.class.name}[#{experiment.run_key.slice(0, 8)}]"
166
+ end
167
+
168
+ def execution_stack
169
+ ActiveSupport::IsolatedExecutionState[:active_experiment_log_subscriber_execution_stack] ||= []
170
+ end
171
+
172
+ def logger
173
+ ActiveExperiment.logger
174
+ end
175
+ end
176
+ end
177
+
178
+ ActiveExperiment::LogSubscriber.attach_to(:active_experiment)
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveExperiment
4
+ # == Logging Module
5
+ #
6
+ # Within experiments, you can log information about the experiment. This can
7
+ # be done by simply calling +logger+ within the experiment definition. By
8
+ # default the logger will be tagged with "ActiveExperiment" to help identify
9
+ # where the log message is coming from.
10
+ #
11
+ # The logger can also be tagged, and then used from within a block. The tag
12
+ # will be removed after the block has been executed.
13
+ #
14
+ # For instance, an experiment could implement tag the logger like this when
15
+ # run:
16
+ #
17
+ # class MyExperiment < ActiveExperiment::Base
18
+ # around_run(prepend: true) do |_, block|
19
+ # tag_logger("MyExperiment", "run", &block)
20
+ # end
21
+ # end
22
+ #
23
+ # Any logging that occurs within this experiment, while it's being run, will
24
+ # be tagged with "MyExperiment" and "run". Those log lines might look like:
25
+ #
26
+ # "[ActiveExperiment] [MyExperiment] [run] Experiment started..."
27
+ #
28
+ # The +log_context+ class attribute is used as configuration within the
29
+ # +ActiveExperiment::LogSubscriber+. For some experiments that contain
30
+ # sensitive information, it might be useful to not log the context. This can
31
+ # be done by setting that experiment class's +log_context+ to false.
32
+ #
33
+ # More logging details can be found in the +ActiveExperiment::LogSubscriber+.
34
+ module Logging # :nodoc:
35
+ extend ActiveSupport::Concern
36
+
37
+ TAG_NAME = "ActiveExperiment"
38
+ private_constant :TAG_NAME
39
+
40
+ included do
41
+ class_attribute :log_context, instance_predicate: true, default: false
42
+ private :log_context=, :log_context
43
+ end
44
+
45
+ # Returns logger that can be used within the experiment.
46
+ #
47
+ # The logger will be tagged with "ActiveExperiment" if possible, to help
48
+ # identify where the log messages are coming from.
49
+ def logger
50
+ @logger ||= ActiveExperiment.logger.try(:tagged, TAG_NAME) || ActiveExperiment.logger
51
+ end
52
+
53
+ private
54
+ def tag_logger(*tags, &block)
55
+ if logger.respond_to?(:tagged)
56
+ logger.tagged(*tags, &block)
57
+ else
58
+ yield
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "global_id/railtie"
4
+ require "active_experiment"
5
+
6
+ module ActiveExperiment
7
+ class Railtie < Rails::Railtie # :nodoc:
8
+ config.active_experiment = ActiveSupport::OrderedOptions.new
9
+ config.active_experiment.custom_rollouts = {}
10
+ config.active_experiment.log_query_tags_around_run = true
11
+
12
+ initializer "active_experiment.logger" do
13
+ ActiveSupport.on_load(:active_experiment) { ActiveExperiment.logger = ::Rails.logger }
14
+ end
15
+
16
+ initializer "active_experiment.custom_rollouts" do |app|
17
+ config.after_initialize do
18
+ app.config.active_experiment.custom_rollouts.each do |name, rollout|
19
+ ActiveExperiment::Rollouts.register(name, rollout)
20
+ end
21
+ end
22
+ end
23
+
24
+ initializer "active_experiment.set_configs" do |app|
25
+ options = app.config.active_experiment
26
+ config.after_initialize do
27
+ options.digest_secret_key ||= app.secrets.secret_key_base
28
+
29
+ options.each do |k, v|
30
+ k = "#{k}="
31
+ if ActiveExperiment.respond_to?(k)
32
+ ActiveExperiment.send(k, v)
33
+ end
34
+ end
35
+ end
36
+
37
+ ActiveSupport.on_load(:active_experiment) do
38
+ options.each do |k, v|
39
+ k = "#{k}="
40
+ if ActiveExperiment.respond_to?(k)
41
+ ActiveExperiment.send(k, v)
42
+ elsif respond_to?(k)
43
+ send(k, v)
44
+ end
45
+ end
46
+ end
47
+
48
+ ActiveSupport.on_load(:action_dispatch_integration_test) do
49
+ include ActiveExperiment::TestHelper
50
+ end
51
+ end
52
+
53
+ initializer "active_experiment.query_log_tags" do |app|
54
+ query_logs_tags_enabled = app.config.respond_to?(:active_record) &&
55
+ app.config.active_record.query_log_tags_enabled &&
56
+ app.config.active_experiment.log_query_tags_around_run
57
+
58
+ if query_logs_tags_enabled
59
+ app.config.active_record.query_log_tags |= [:experiment]
60
+
61
+ ActiveSupport.on_load(:active_record) do
62
+ ActiveRecord::QueryLogs.taggings[:experiment] = lambda do |context|
63
+ context[:experiment].class.name if context[:experiment]
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end