split 3.3.2 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.eslintrc +1 -1
- data/.github/FUNDING.yml +1 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +24 -0
- data/.github/dependabot.yml +7 -0
- data/.github/workflows/ci.yml +63 -0
- data/.rspec +1 -0
- data/.rubocop.yml +67 -1043
- data/CHANGELOG.md +121 -0
- data/CODE_OF_CONDUCT.md +3 -3
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +6 -1
- data/README.md +51 -21
- data/Rakefile +6 -5
- data/lib/split/algorithms/block_randomization.rb +7 -6
- data/lib/split/algorithms/weighted_sample.rb +2 -1
- data/lib/split/algorithms/whiplash.rb +17 -18
- data/lib/split/algorithms.rb +14 -0
- data/lib/split/alternative.rb +25 -25
- data/lib/split/cache.rb +27 -0
- data/lib/split/combined_experiments_helper.rb +5 -4
- data/lib/split/configuration.rb +94 -96
- data/lib/split/dashboard/helpers.rb +7 -7
- data/lib/split/dashboard/pagination_helpers.rb +56 -57
- data/lib/split/dashboard/paginator.rb +1 -0
- data/lib/split/dashboard/public/dashboard.js +10 -0
- data/lib/split/dashboard/public/style.css +10 -2
- data/lib/split/dashboard/views/_controls.erb +13 -0
- data/lib/split/dashboard/views/_experiment.erb +2 -1
- data/lib/split/dashboard/views/index.erb +19 -4
- data/lib/split/dashboard/views/layout.erb +1 -1
- data/lib/split/dashboard.rb +46 -21
- data/lib/split/encapsulated_helper.rb +15 -8
- data/lib/split/engine.rb +7 -4
- data/lib/split/exceptions.rb +1 -0
- data/lib/split/experiment.rb +160 -122
- data/lib/split/experiment_catalog.rb +7 -8
- data/lib/split/extensions/string.rb +2 -1
- data/lib/split/goals_collection.rb +10 -10
- data/lib/split/helper.rb +52 -24
- data/lib/split/metric.rb +6 -6
- data/lib/split/persistence/cookie_adapter.rb +47 -44
- data/lib/split/persistence/dual_adapter.rb +53 -12
- data/lib/split/persistence/redis_adapter.rb +8 -4
- data/lib/split/persistence/session_adapter.rb +1 -2
- data/lib/split/persistence.rb +8 -6
- data/lib/split/redis_interface.rb +16 -29
- data/lib/split/trial.rb +44 -35
- data/lib/split/user.rb +30 -15
- data/lib/split/version.rb +2 -4
- data/lib/split/zscore.rb +2 -3
- data/lib/split.rb +35 -28
- data/spec/algorithms/block_randomization_spec.rb +6 -5
- data/spec/algorithms/weighted_sample_spec.rb +6 -5
- data/spec/algorithms/whiplash_spec.rb +4 -5
- data/spec/alternative_spec.rb +35 -36
- data/spec/cache_spec.rb +84 -0
- data/spec/combined_experiments_helper_spec.rb +18 -17
- data/spec/configuration_spec.rb +41 -45
- data/spec/dashboard/pagination_helpers_spec.rb +71 -67
- data/spec/dashboard/paginator_spec.rb +10 -9
- data/spec/dashboard_helpers_spec.rb +19 -18
- data/spec/dashboard_spec.rb +153 -48
- data/spec/encapsulated_helper_spec.rb +47 -23
- data/spec/experiment_catalog_spec.rb +14 -13
- data/spec/experiment_spec.rb +224 -111
- data/spec/goals_collection_spec.rb +18 -16
- data/spec/helper_spec.rb +531 -424
- data/spec/metric_spec.rb +14 -14
- data/spec/persistence/cookie_adapter_spec.rb +26 -11
- data/spec/persistence/dual_adapter_spec.rb +158 -66
- data/spec/persistence/redis_adapter_spec.rb +35 -27
- data/spec/persistence/session_adapter_spec.rb +2 -3
- data/spec/persistence_spec.rb +1 -2
- data/spec/redis_interface_spec.rb +25 -82
- data/spec/spec_helper.rb +38 -24
- data/spec/split_spec.rb +11 -11
- data/spec/support/cookies_mock.rb +1 -2
- data/spec/trial_spec.rb +102 -75
- data/spec/user_spec.rb +69 -27
- data/split.gemspec +26 -23
- metadata +68 -42
- data/.travis.yml +0 -66
- data/Appraisals +0 -19
- data/gemfiles/4.2.gemfile +0 -9
- data/gemfiles/5.0.gemfile +0 -9
- data/gemfiles/5.1.gemfile +0 -9
- data/gemfiles/5.2.gemfile +0 -9
- data/gemfiles/6.0.gemfile +0 -9
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,124 @@
|
|
|
1
|
+
# 4.0.5 (Aug 3rd, 2025)
|
|
2
|
+
|
|
3
|
+
Bugfixes:
|
|
4
|
+
- Handle when Rails is partially loaded as a Gem. (@bjacobs09, #727)
|
|
5
|
+
- Fix Rack compatibility with versions > 3 (@andrehjr, #729)
|
|
6
|
+
|
|
7
|
+
Misc:
|
|
8
|
+
- Add funding_uri to gemspec (@andrew, #726)
|
|
9
|
+
- Drop Rails 5.x Support (@andrehjr, #728)
|
|
10
|
+
- Drop Rails 6.0 and Ruby < 2.6 (@andrehjr, #729)
|
|
11
|
+
- Add support for Ruby 3.5+ (@andrehjr, #737)
|
|
12
|
+
|
|
13
|
+
# 4.0.4 (March 3rd, 2024)
|
|
14
|
+
|
|
15
|
+
Bugfixes:
|
|
16
|
+
- Better integration for EncapsulatedHelper when needing params/request info (@henrique-ft, #721 and #723)
|
|
17
|
+
|
|
18
|
+
Misc:
|
|
19
|
+
- Make specs compatible with newer Rack versions (@andrehjr, #722)
|
|
20
|
+
|
|
21
|
+
# 4.0.3 (November 15th, 2023)
|
|
22
|
+
|
|
23
|
+
Bugfixes:
|
|
24
|
+
- Do not throw error if alternativas have data that can lead to negative numbers for probability calculation (@andrehjr, #703)
|
|
25
|
+
- Do not persist invalid extra_info on ab_record_extra_info. (@trostli @andrehjr, #717)
|
|
26
|
+
- CROSSSLOT keys issue fix when using redis cluster (@naveen-chidhambaram, #710)
|
|
27
|
+
- Convert value to string before saving it in RedisAdapter (@Jealrock, #714)
|
|
28
|
+
- Fix deprecation warning with Redis 4.8.0 (@martingregoire, #701)
|
|
29
|
+
|
|
30
|
+
Misc:
|
|
31
|
+
- Add matrix as a default dependency (@andrehjr, #705)
|
|
32
|
+
- Add Ruby 3.2 to Github Actions (@andrehjr, #702)
|
|
33
|
+
- Update documentation regarding finding users outside a web session (@andrehjr, #716)
|
|
34
|
+
- Update actions/checkout to v4 (@andrehjr, #718)
|
|
35
|
+
|
|
36
|
+
# 4.0.2 (December 2nd, 2022)
|
|
37
|
+
|
|
38
|
+
Bugfixes:
|
|
39
|
+
- Stop crashing on non-hash json (@knarewski, #697)
|
|
40
|
+
- Handle when Rails is partially loaded as a Gem (@TSMMark, #687)
|
|
41
|
+
|
|
42
|
+
Features:
|
|
43
|
+
- Add support for redis-client, which does not automatically cast types to strings (@knarewski, #696)
|
|
44
|
+
- Add ability to initialize experiments (@robin-phung, #673)
|
|
45
|
+
|
|
46
|
+
Misc:
|
|
47
|
+
- Fix default branch name and gem metadata indentation (@ursm, #693)
|
|
48
|
+
- Update actions/checkout to v3 (@andrehjr, #683)
|
|
49
|
+
- Enforce double quotes (@andrehjr, #682)
|
|
50
|
+
- Fix Rubocop Style/* Offenses (@andrehjr, #681)
|
|
51
|
+
- Enable rubocop on Github Actions (@andrehjr, #680)
|
|
52
|
+
- Fix all Layout issues on the project (@andrehjr, #679)
|
|
53
|
+
- Fix Style/HashSyntax offenses (@andrehjr, #678)
|
|
54
|
+
- Remove usage of deprecated implicit block expectation from specs (@andrehjr, #677)
|
|
55
|
+
- Remove appraisals configuration (@andrehjr, #676)
|
|
56
|
+
- Add Ruby 3.1 (@andrehjr, #675)
|
|
57
|
+
- Encapsulate Split::Algorithms at our own module to avoid explicit calling rubystats everywhere (@andrehjr, #674)
|
|
58
|
+
|
|
59
|
+
## 4.0.1 (December 30th, 2021)
|
|
60
|
+
|
|
61
|
+
Bugfixes:
|
|
62
|
+
- ab_test must return metadata on error or if split is disabled/excluded user (@andrehjr, #622)
|
|
63
|
+
- Fix versioned experiments when used with allow_multiple_experiments=control (@andrehjr, #613)
|
|
64
|
+
- Only block Pinterest bot (@huoxito, #606)
|
|
65
|
+
- Respect experiment defaults when loading experiments in initializer. (@mattwd7, #599)
|
|
66
|
+
- Removes metadata key when it updated to nil (@andrehjr, #633)
|
|
67
|
+
- Force experiment does not count for metrics (@andrehjr, #637)
|
|
68
|
+
- Fix cleanup_old_versions! misbehaviour (@serggl, #661)
|
|
69
|
+
|
|
70
|
+
Features:
|
|
71
|
+
- Make goals accessible via on_trial_complete callbacks (@robin-phung, #625)
|
|
72
|
+
- Replace usage of SimpleRandom with RubyStats(Used for Beta Distribution RNG) (@andrehjr, #616)
|
|
73
|
+
- Introduce enable/disable experiment cohorting (@robin-phung, #615)
|
|
74
|
+
- Add on_experiment_winner_choose callback (@GenaMinenkov, #574)
|
|
75
|
+
- Add Split::Cache to reduce load on Redis (@rdh, #648)
|
|
76
|
+
- Caching based optimization in the experiment#save path (@amangup, #652)
|
|
77
|
+
- Adds config option for cookie domain (@joedelia, #664)
|
|
78
|
+
|
|
79
|
+
Misc:
|
|
80
|
+
- Drop support for Ruby < 2.5 (@andrehjr, #627)
|
|
81
|
+
- Drop support for Rails < 5 (@andrehjr, #607)
|
|
82
|
+
- Bump minimum required redis to 4.2 (@andrehjr, #628)
|
|
83
|
+
- Removed repeated loading from config (@robin-phung, #619)
|
|
84
|
+
- Simplify RedisInterface usage when persisting Experiment alternatives (@andrehjr, #632)
|
|
85
|
+
- Remove redis_url impl. Deprecated on version 2.2 (@andrehjr, #631)
|
|
86
|
+
- Remove thread_safe config as redis-rb is thread_safe by default (@andrehjr, #630)
|
|
87
|
+
- Fix typo of in `Split::Trial` class variable (TomasBarry, #644)
|
|
88
|
+
- Single HSET to update values, instead of multiple ones (@andrehjr, #640)
|
|
89
|
+
- Use Redis#hmset to keep compatibility with Redis < 4.0 (@andrehjr, #659)
|
|
90
|
+
- Remove 'set' parsing for alternatives. Sets were used as storage and deprecated on 0.x (@andrehjr, #639)
|
|
91
|
+
- Adding documentation related to what is stored on cookies. (@andrehjr, #634)
|
|
92
|
+
- Keep railtie defined under the Split gem namespace (@avit, #666)
|
|
93
|
+
- Update RSpec helper to support block syntax (@clowder, #665)
|
|
94
|
+
|
|
95
|
+
## 3.4.1 (November 12th, 2019)
|
|
96
|
+
|
|
97
|
+
Bugfixes:
|
|
98
|
+
- Reference ActionController directly when including split helpers, to avoid breaking Rails API Controllers (@andrehjr, #602)
|
|
99
|
+
|
|
100
|
+
## 3.4.0 (November 9th, 2019)
|
|
101
|
+
|
|
102
|
+
Features:
|
|
103
|
+
- Improve DualAdapter (@santib, #588), adds a new configuration for the DualAdapter, making it possible to keep consistency for logged_out/logged_in users. It's a opt-in flag. No Behavior was changed on this release.
|
|
104
|
+
- Make dashboard pagination default "per" param configurable (@alopatin, #597)
|
|
105
|
+
|
|
106
|
+
Bugfixes:
|
|
107
|
+
- Fix `force_alternative` for experiments with incremented version (@giraffate, #568)
|
|
108
|
+
- Persist alternative weights (@giraffate, #570)
|
|
109
|
+
- Combined experiment performance improvements (@gnanou, #575)
|
|
110
|
+
- Handle correctly case when ab_finished is called before ab_test for a user (@gnanou, #577)
|
|
111
|
+
- When loading active_experiments, it should not look into user's 'finished' keys (@andrehjr, #582)
|
|
112
|
+
|
|
113
|
+
Misc:
|
|
114
|
+
- Remove `rubyforge_project` from gemspec (@giraffate, #583)
|
|
115
|
+
- Fix URLs to replace http with https (@giraffate , #584)
|
|
116
|
+
- Lazily include split helpers in ActionController::Base (@hasghari, #586)
|
|
117
|
+
- Fix unused variable warnings (@andrehjr, #592)
|
|
118
|
+
- Fix ruby warnings (@andrehjr, #593)
|
|
119
|
+
- Update rubocop.yml config (@andrehjr, #594)
|
|
120
|
+
- Add frozen_string_literal to all files that were missing it (@andrehjr, #595)
|
|
121
|
+
|
|
1
122
|
## 3.3.2 (April 12th, 2019)
|
|
2
123
|
|
|
3
124
|
Features:
|
data/CODE_OF_CONDUCT.md
CHANGED
|
@@ -68,7 +68,7 @@ members of the project's leadership.
|
|
|
68
68
|
## Attribution
|
|
69
69
|
|
|
70
70
|
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
|
71
|
-
available at [
|
|
71
|
+
available at [https://contributor-covenant.org/version/1/4][version]
|
|
72
72
|
|
|
73
|
-
[homepage]:
|
|
74
|
-
[version]:
|
|
73
|
+
[homepage]: https://contributor-covenant.org
|
|
74
|
+
[version]: https://contributor-covenant.org/version/1/4/
|
data/CONTRIBUTING.md
CHANGED
|
@@ -25,7 +25,7 @@ Want to contribute to Split? That's great! Here are a couple of guidelines that
|
|
|
25
25
|
|
|
26
26
|
## Setup instructions
|
|
27
27
|
|
|
28
|
-
You can find in-depth instructions to install in our [README](https://github.com/splitrb/split/blob/
|
|
28
|
+
You can find in-depth instructions to install in our [README](https://github.com/splitrb/split/blob/main/README.md).
|
|
29
29
|
|
|
30
30
|
*Note*: Split requires Ruby 1.9.2 or higher.
|
|
31
31
|
|
data/Gemfile
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source "https://rubygems.org"
|
|
2
4
|
|
|
3
5
|
gemspec
|
|
4
6
|
|
|
5
|
-
gem "
|
|
7
|
+
gem "rubocop", require: false
|
|
6
8
|
gem "codeclimate-test-reporter"
|
|
9
|
+
gem "concurrent-ruby", "< 1.3.5"
|
|
10
|
+
|
|
11
|
+
gem "rails", "~> #{ENV.fetch('RAILS_VERSION', '8.0')}"
|
data/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
# [Split](
|
|
1
|
+
# [Split](https://libraries.io/rubygems/split)
|
|
2
2
|
|
|
3
3
|
[](http://badge.fury.io/rb/split)
|
|
4
|
-
|
|
4
|
+

|
|
5
5
|
[](https://codeclimate.com/github/splitrb/split)
|
|
6
6
|
[](https://codeclimate.com/github/splitrb/split/coverage)
|
|
7
7
|
[](https://github.com/RichardLitt/standard-readme)
|
|
8
8
|
[](https://www.codetriage.com/splitrb/split)
|
|
9
9
|
|
|
10
|
-
> 📈 The Rack Based A/B testing framework
|
|
10
|
+
> 📈 The Rack Based A/B testing framework https://libraries.io/rubygems/split
|
|
11
11
|
|
|
12
12
|
Split is a rack based A/B testing framework designed to work with Rails, Sinatra or any other rack based app.
|
|
13
13
|
|
|
@@ -19,11 +19,13 @@ Split is designed to be hacker friendly, allowing for maximum customisation and
|
|
|
19
19
|
|
|
20
20
|
### Requirements
|
|
21
21
|
|
|
22
|
-
Split
|
|
22
|
+
Split v4.0+ is currently tested with Ruby >= 2.5 and Rails >= 5.2.
|
|
23
|
+
|
|
24
|
+
If your project requires compatibility with Ruby 2.4.x or older Rails versions. You can try v3.0 or v0.8.0(for Ruby 1.9.3)
|
|
23
25
|
|
|
24
26
|
Split uses Redis as a datastore.
|
|
25
27
|
|
|
26
|
-
Split only supports Redis
|
|
28
|
+
Split only supports Redis 4.0 or greater.
|
|
27
29
|
|
|
28
30
|
If you're on OS X, Homebrew is the simplest way to install Redis:
|
|
29
31
|
|
|
@@ -110,9 +112,9 @@ Split has two options for you to use to determine which alternative is the best.
|
|
|
110
112
|
|
|
111
113
|
The first option (default on the dashboard) uses a z test (n>30) for the difference between your control and alternative conversion rates to calculate statistical significance. This test will tell you whether an alternative is better or worse than your control, but it will not distinguish between which alternative is the best in an experiment with multiple alternatives. Split will only tell you if your experiment is 90%, 95%, or 99% significant, and this test only works if you have more than 30 participants and 5 conversions for each branch.
|
|
112
114
|
|
|
113
|
-
As per this [blog post](
|
|
115
|
+
As per this [blog post](https://www.evanmiller.org/how-not-to-run-an-ab-test.html) on the pitfalls of A/B testing, it is highly recommended that you determine your requisite sample size for each branch before running the experiment. Otherwise, you'll have an increased rate of false positives (experiments which show a significant effect where really there is none).
|
|
114
116
|
|
|
115
|
-
[Here](
|
|
117
|
+
[Here](https://www.evanmiller.org/ab-testing/sample-size.html) is a sample size calculator for your convenience.
|
|
116
118
|
|
|
117
119
|
The second option uses simulations from a beta distribution to determine the probability that the given alternative is the winner compared to all other alternatives. You can view these probabilities by clicking on the drop-down menu labeled "Confidence." This option should be used when the experiment has more than just 1 control and 1 alternative. It can also be used for a simple, 2-alternative A/B test.
|
|
118
120
|
|
|
@@ -175,8 +177,10 @@ module SplitHelper
|
|
|
175
177
|
# use_ab_test(signup_form: "single_page", pricing: "show_enterprise_prices")
|
|
176
178
|
#
|
|
177
179
|
def use_ab_test(alternatives_by_experiment)
|
|
178
|
-
allow_any_instance_of(Split::Helper).to receive(:ab_test) do |_receiver, experiment|
|
|
179
|
-
alternatives_by_experiment.fetch(experiment) { |key| raise "Unknown experiment '#{key}'" }
|
|
180
|
+
allow_any_instance_of(Split::Helper).to receive(:ab_test) do |_receiver, experiment, &block|
|
|
181
|
+
variant = alternatives_by_experiment.fetch(experiment) { |key| raise "Unknown experiment '#{key}'" }
|
|
182
|
+
block.call(variant) unless block.nil?
|
|
183
|
+
variant
|
|
180
184
|
end
|
|
181
185
|
end
|
|
182
186
|
end
|
|
@@ -263,7 +267,7 @@ Split.configure do |config|
|
|
|
263
267
|
end
|
|
264
268
|
```
|
|
265
269
|
|
|
266
|
-
|
|
270
|
+
When using the cookie persistence, Split stores data into an anonymous tracking cookie named 'split', which expires in 1 year. To change that, set the `persistence_cookie_length` in the configuration (unit of time in seconds).
|
|
267
271
|
|
|
268
272
|
```ruby
|
|
269
273
|
Split.configure do |config|
|
|
@@ -272,6 +276,8 @@ Split.configure do |config|
|
|
|
272
276
|
end
|
|
273
277
|
```
|
|
274
278
|
|
|
279
|
+
The data stored consists of the experiment name and the variants the user is in. Example: { "experiment_name" => "variant_a" }
|
|
280
|
+
|
|
275
281
|
__Note:__ Using cookies depends on `ActionDispatch::Cookies` or any identical API
|
|
276
282
|
|
|
277
283
|
#### Redis
|
|
@@ -360,7 +366,7 @@ end
|
|
|
360
366
|
|
|
361
367
|
If you are running `ab_test` from a view, you must define your event
|
|
362
368
|
hook callback as a
|
|
363
|
-
[helper_method](
|
|
369
|
+
[helper_method](https://apidock.com/rails/AbstractController/Helpers/ClassMethods/helper_method)
|
|
364
370
|
in the controller:
|
|
365
371
|
|
|
366
372
|
``` ruby
|
|
@@ -386,6 +392,8 @@ Split.configure do |config|
|
|
|
386
392
|
# before experiment reset or deleted
|
|
387
393
|
config.on_before_experiment_reset = -> (example) { # Do something on reset }
|
|
388
394
|
config.on_before_experiment_delete = -> (experiment) { # Do something else on delete }
|
|
395
|
+
# after experiment winner had been set
|
|
396
|
+
config.on_experiment_winner_choose = -> (experiment) { # Do something on winner choose }
|
|
389
397
|
end
|
|
390
398
|
```
|
|
391
399
|
|
|
@@ -446,7 +454,7 @@ match "/split" => Split::Dashboard, anchor: false, via: [:get, :post, :delete],
|
|
|
446
454
|
end
|
|
447
455
|
```
|
|
448
456
|
|
|
449
|
-
More information on this [here](
|
|
457
|
+
More information on this [here](https://steve.dynedge.co.uk/2011/12/09/controlling-access-to-routes-and-rack-apps-in-rails-3-with-devise-and-warden/)
|
|
450
458
|
|
|
451
459
|
### Screenshot
|
|
452
460
|
|
|
@@ -556,7 +564,7 @@ and:
|
|
|
556
564
|
ab_finished(:my_first_experiment)
|
|
557
565
|
```
|
|
558
566
|
|
|
559
|
-
You can also add meta data for each experiment, very useful when you need more than an alternative name to change behaviour:
|
|
567
|
+
You can also add meta data for each experiment, which is very useful when you need more than an alternative name to change behaviour:
|
|
560
568
|
|
|
561
569
|
```ruby
|
|
562
570
|
Split.configure do |config|
|
|
@@ -601,6 +609,8 @@ or in views:
|
|
|
601
609
|
<% end %>
|
|
602
610
|
```
|
|
603
611
|
|
|
612
|
+
The keys used in meta data should be Strings
|
|
613
|
+
|
|
604
614
|
#### Metrics
|
|
605
615
|
|
|
606
616
|
You might wish to track generic metrics, such as conversions, and use
|
|
@@ -642,7 +652,7 @@ The API to define goals for an experiment is this:
|
|
|
642
652
|
ab_test({link_color: ["purchase", "refund"]}, "red", "blue")
|
|
643
653
|
```
|
|
644
654
|
|
|
645
|
-
or you can
|
|
655
|
+
or you can define them in a configuration file:
|
|
646
656
|
|
|
647
657
|
```ruby
|
|
648
658
|
Split.configure do |config|
|
|
@@ -750,6 +760,20 @@ split_config = YAML.load_file(Rails.root.join('config', 'split.yml'))
|
|
|
750
760
|
Split.redis = split_config[Rails.env]
|
|
751
761
|
```
|
|
752
762
|
|
|
763
|
+
### Redis Caching (v4.0+)
|
|
764
|
+
|
|
765
|
+
In some high-volume usage scenarios, Redis load can be incurred by repeated
|
|
766
|
+
fetches for fairly static data. Enabling caching will reduce this load.
|
|
767
|
+
|
|
768
|
+
```ruby
|
|
769
|
+
Split.configuration.cache = true
|
|
770
|
+
````
|
|
771
|
+
|
|
772
|
+
This currently caches:
|
|
773
|
+
- `Split::ExperimentCatalog.find`
|
|
774
|
+
- `Split::Experiment.start_time`
|
|
775
|
+
- `Split::Experiment.winner`
|
|
776
|
+
|
|
753
777
|
## Namespaces
|
|
754
778
|
|
|
755
779
|
If you're running multiple, separate instances of Split you may want
|
|
@@ -766,7 +790,7 @@ library. To configure Split to use `Redis::Namespace`, do the following:
|
|
|
766
790
|
```
|
|
767
791
|
|
|
768
792
|
2. Configure `Split.redis` to use a `Redis::Namespace` instance (possible in an
|
|
769
|
-
|
|
793
|
+
initializer):
|
|
770
794
|
|
|
771
795
|
```ruby
|
|
772
796
|
redis = Redis.new(url: ENV['REDIS_URL']) # or whatever config you want
|
|
@@ -783,10 +807,16 @@ conduct experiments that are not tied to a web session.
|
|
|
783
807
|
```ruby
|
|
784
808
|
# create a new experiment
|
|
785
809
|
experiment = Split::ExperimentCatalog.find_or_create('color', 'red', 'blue')
|
|
810
|
+
|
|
811
|
+
# find the user
|
|
812
|
+
user = Split::User.find(user_id, :redis)
|
|
813
|
+
|
|
786
814
|
# create a new trial
|
|
787
|
-
trial = Split::Trial.new(:
|
|
815
|
+
trial = Split::Trial.new(user: user, experiment: experiment)
|
|
816
|
+
|
|
788
817
|
# run trial
|
|
789
818
|
trial.choose!
|
|
819
|
+
|
|
790
820
|
# get the result, returns either red or blue
|
|
791
821
|
trial.alternative.name
|
|
792
822
|
|
|
@@ -824,8 +854,8 @@ end
|
|
|
824
854
|
|
|
825
855
|
## Extensions
|
|
826
856
|
|
|
827
|
-
- [Split::Export](
|
|
828
|
-
- [Split::Analytics](
|
|
857
|
+
- [Split::Export](https://github.com/splitrb/split-export) - Easily export A/B test data out of Split.
|
|
858
|
+
- [Split::Analytics](https://github.com/splitrb/split-analytics) - Push test data to Google Analytics.
|
|
829
859
|
- [Split::Mongoid](https://github.com/MongoHQ/split-mongoid) - Store experiment data in mongoid (still uses redis).
|
|
830
860
|
- [Split::Cacheable](https://github.com/harrystech/split_cacheable) - Automatically create cache buckets per test.
|
|
831
861
|
- [Split::Counters](https://github.com/bernardkroes/split-counters) - Add counters per experiment and alternative.
|
|
@@ -837,7 +867,7 @@ Ryan bates has produced an excellent 10 minute screencast about split on the Rai
|
|
|
837
867
|
|
|
838
868
|
## Blogposts
|
|
839
869
|
|
|
840
|
-
* [Recipe: A/B testing with KISSMetrics and the split gem](
|
|
870
|
+
* [Recipe: A/B testing with KISSMetrics and the split gem](https://robots.thoughtbot.com/post/9595887299/recipe-a-b-testing-with-kissmetrics-and-the-split-gem)
|
|
841
871
|
* [Rails A/B testing with Split on Heroku](http://blog.nathanhumbert.com/2012/02/rails-ab-testing-with-split-on-heroku.html)
|
|
842
872
|
|
|
843
873
|
## Backers
|
|
@@ -917,9 +947,9 @@ Please do! Over 70 different people have contributed to the project, you can see
|
|
|
917
947
|
|
|
918
948
|
### Development
|
|
919
949
|
|
|
920
|
-
The source code is hosted at [GitHub](
|
|
950
|
+
The source code is hosted at [GitHub](https://github.com/splitrb/split).
|
|
921
951
|
|
|
922
|
-
Report issues and feature requests on [GitHub Issues](
|
|
952
|
+
Report issues and feature requests on [GitHub Issues](https://github.com/splitrb/split/issues).
|
|
923
953
|
|
|
924
954
|
You can find a discussion form on [Google Groups](https://groups.google.com/d/forum/split-ruby).
|
|
925
955
|
|
data/Rakefile
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env rake
|
|
2
|
-
|
|
3
|
-
require 'rspec/core/rake_task'
|
|
4
|
-
require 'appraisal'
|
|
2
|
+
# frozen_string_literal: true
|
|
5
3
|
|
|
6
|
-
|
|
4
|
+
require "bundler/gem_tasks"
|
|
5
|
+
require "rspec/core/rake_task"
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
RSpec::Core::RakeTask.new("spec")
|
|
8
|
+
|
|
9
|
+
task default: :spec
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# Selects alternative with minimum count of participants
|
|
2
4
|
# If all counts are even (i.e. all are minimum), samples from all possible alternatives
|
|
3
5
|
|
|
@@ -10,12 +12,11 @@ module Split
|
|
|
10
12
|
end
|
|
11
13
|
|
|
12
14
|
private
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
end
|
|
15
|
+
def minimum_participant_alternatives(alternatives)
|
|
16
|
+
alternatives_by_count = alternatives.group_by(&:participant_count)
|
|
17
|
+
min_group = alternatives_by_count.min_by { |k, v| k }
|
|
18
|
+
min_group.last
|
|
19
|
+
end
|
|
19
20
|
end
|
|
20
21
|
end
|
|
21
22
|
end
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
module Split
|
|
3
4
|
module Algorithms
|
|
4
5
|
module WeightedSample
|
|
@@ -8,7 +9,7 @@ module Split
|
|
|
8
9
|
total = weights.inject(:+)
|
|
9
10
|
point = rand * total
|
|
10
11
|
|
|
11
|
-
experiment.alternatives.zip(weights).each do |n,w|
|
|
12
|
+
experiment.alternatives.zip(weights).each do |n, w|
|
|
12
13
|
return n if w >= point
|
|
13
14
|
point -= w
|
|
14
15
|
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
# A multi-armed bandit implementation inspired by
|
|
3
4
|
# @aaronsw and victorykit/whiplash
|
|
4
|
-
require 'simple-random'
|
|
5
5
|
|
|
6
6
|
module Split
|
|
7
7
|
module Algorithms
|
|
@@ -12,26 +12,25 @@ module Split
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
private
|
|
15
|
+
def arm_guess(participants, completions)
|
|
16
|
+
a = [participants, 0].max
|
|
17
|
+
b = [participants-completions, 0].max
|
|
18
|
+
Split::Algorithms.beta_distribution_rng(a + fairness_constant, b + fairness_constant)
|
|
19
|
+
end
|
|
15
20
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
alternatives.each do |alternative|
|
|
25
|
-
guesses[alternative.name] = arm_guess(alternative.participant_count, alternative.all_completed_count)
|
|
21
|
+
def best_guess(alternatives)
|
|
22
|
+
guesses = {}
|
|
23
|
+
alternatives.each do |alternative|
|
|
24
|
+
guesses[alternative.name] = arm_guess(alternative.participant_count, alternative.all_completed_count)
|
|
25
|
+
end
|
|
26
|
+
gmax = guesses.values.max
|
|
27
|
+
best = guesses.keys.select { |name| guesses[name] == gmax }
|
|
28
|
+
best.sample
|
|
26
29
|
end
|
|
27
|
-
gmax = guesses.values.max
|
|
28
|
-
best = guesses.keys.select { |name| guesses[name] == gmax }
|
|
29
|
-
best.sample
|
|
30
|
-
end
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
31
|
+
def fairness_constant
|
|
32
|
+
7
|
|
33
|
+
end
|
|
35
34
|
end
|
|
36
35
|
end
|
|
37
36
|
end
|
data/lib/split/alternative.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
module Split
|
|
3
4
|
class Alternative
|
|
4
5
|
attr_accessor :name
|
|
@@ -15,7 +16,7 @@ module Split
|
|
|
15
16
|
@name = name
|
|
16
17
|
@weight = 1
|
|
17
18
|
end
|
|
18
|
-
p_winner = 0.0
|
|
19
|
+
@p_winner = 0.0
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def to_s
|
|
@@ -37,11 +38,11 @@ module Split
|
|
|
37
38
|
end
|
|
38
39
|
|
|
39
40
|
def participant_count
|
|
40
|
-
Split.redis.hget(key,
|
|
41
|
+
Split.redis.hget(key, "participant_count").to_i
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
def participant_count=(count)
|
|
44
|
-
Split.redis.hset(key,
|
|
45
|
+
Split.redis.hset(key, "participant_count", count.to_i)
|
|
45
46
|
end
|
|
46
47
|
|
|
47
48
|
def completed_count(goal = nil)
|
|
@@ -66,22 +67,22 @@ module Split
|
|
|
66
67
|
def set_field(goal)
|
|
67
68
|
field = "completed_count"
|
|
68
69
|
field += ":" + goal unless goal.nil?
|
|
69
|
-
|
|
70
|
+
field
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
def set_prob_field(goal)
|
|
73
74
|
field = "p_winner"
|
|
74
75
|
field += ":" + goal unless goal.nil?
|
|
75
|
-
|
|
76
|
+
field
|
|
76
77
|
end
|
|
77
78
|
|
|
78
|
-
def set_completed_count
|
|
79
|
+
def set_completed_count(count, goal = nil)
|
|
79
80
|
field = set_field(goal)
|
|
80
81
|
Split.redis.hset(key, field, count.to_i)
|
|
81
82
|
end
|
|
82
83
|
|
|
83
84
|
def increment_participation
|
|
84
|
-
Split.redis.hincrby key,
|
|
85
|
+
Split.redis.hincrby key, "participant_count", 1
|
|
85
86
|
end
|
|
86
87
|
|
|
87
88
|
def increment_completion(goal = nil)
|
|
@@ -111,7 +112,7 @@ module Split
|
|
|
111
112
|
control = experiment.control
|
|
112
113
|
alternative = self
|
|
113
114
|
|
|
114
|
-
return
|
|
115
|
+
return "N/A" if control.name == alternative.name
|
|
115
116
|
|
|
116
117
|
p_a = alternative.conversion_rate(goal)
|
|
117
118
|
p_c = control.conversion_rate(goal)
|
|
@@ -120,13 +121,13 @@ module Split
|
|
|
120
121
|
n_c = control.participant_count
|
|
121
122
|
|
|
122
123
|
# can't calculate zscore for P(x) > 1
|
|
123
|
-
return
|
|
124
|
+
return "N/A" if p_a > 1 || p_c > 1
|
|
124
125
|
|
|
125
|
-
|
|
126
|
+
Split::Zscore.calculate(p_a, n_a, p_c, n_c)
|
|
126
127
|
end
|
|
127
128
|
|
|
128
129
|
def extra_info
|
|
129
|
-
data = Split.redis.hget(key,
|
|
130
|
+
data = Split.redis.hget(key, "recorded_info")
|
|
130
131
|
if data && data.length > 1
|
|
131
132
|
begin
|
|
132
133
|
JSON.parse(data)
|
|
@@ -148,24 +149,24 @@ module Split
|
|
|
148
149
|
@recorded_info[k] = value
|
|
149
150
|
end
|
|
150
151
|
|
|
151
|
-
Split.redis.hset key,
|
|
152
|
+
Split.redis.hset key, "recorded_info", (@recorded_info || {}).to_json
|
|
152
153
|
end
|
|
153
154
|
|
|
154
155
|
def save
|
|
155
|
-
Split.redis.hsetnx key,
|
|
156
|
-
Split.redis.hsetnx key,
|
|
157
|
-
Split.redis.hsetnx key,
|
|
158
|
-
Split.redis.hsetnx key,
|
|
156
|
+
Split.redis.hsetnx key, "participant_count", 0
|
|
157
|
+
Split.redis.hsetnx key, "completed_count", 0
|
|
158
|
+
Split.redis.hsetnx key, "p_winner", p_winner
|
|
159
|
+
Split.redis.hsetnx key, "recorded_info", (@recorded_info || {}).to_json
|
|
159
160
|
end
|
|
160
161
|
|
|
161
162
|
def validate!
|
|
162
163
|
unless String === @name || hash_with_correct_values?(@name)
|
|
163
|
-
raise ArgumentError,
|
|
164
|
+
raise ArgumentError, "Alternative must be a string"
|
|
164
165
|
end
|
|
165
166
|
end
|
|
166
167
|
|
|
167
168
|
def reset
|
|
168
|
-
Split.redis.hmset key,
|
|
169
|
+
Split.redis.hmset key, "participant_count", 0, "completed_count", 0, "recorded_info", ""
|
|
169
170
|
unless goals.empty?
|
|
170
171
|
goals.each do |g|
|
|
171
172
|
field = "completed_count:#{g}"
|
|
@@ -179,13 +180,12 @@ module Split
|
|
|
179
180
|
end
|
|
180
181
|
|
|
181
182
|
private
|
|
183
|
+
def hash_with_correct_values?(name)
|
|
184
|
+
Hash === name && String === name.keys.first && Float(name.values.first) rescue false
|
|
185
|
+
end
|
|
182
186
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
def key
|
|
188
|
-
"#{experiment_name}:#{name}"
|
|
189
|
-
end
|
|
187
|
+
def key
|
|
188
|
+
"#{experiment_name}:#{name}"
|
|
189
|
+
end
|
|
190
190
|
end
|
|
191
191
|
end
|
data/lib/split/cache.rb
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Split
|
|
4
|
+
class Cache
|
|
5
|
+
def self.clear
|
|
6
|
+
@cache = nil
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def self.fetch(namespace, key)
|
|
10
|
+
return yield unless Split.configuration.cache
|
|
11
|
+
|
|
12
|
+
@cache ||= {}
|
|
13
|
+
@cache[namespace] ||= {}
|
|
14
|
+
|
|
15
|
+
value = @cache[namespace][key]
|
|
16
|
+
return value if value
|
|
17
|
+
|
|
18
|
+
@cache[namespace][key] = yield
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.clear_key(key)
|
|
22
|
+
@cache&.keys&.each do |namespace|
|
|
23
|
+
@cache[namespace]&.delete(key)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|