field_test 0.5.0 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 913c2ad9eaefe134cdabd915eec17f4f48fa9062c9066e37aa638989f0c05072
-  data.tar.gz: bfe73138f6b18fbf8b72e144d2f3c6e727a1d398e6c0f09860981aade30b9d45
+  metadata.gz: fdb76c20b686424fab97716c260d114977a33ca9d9b41f8ba03a25523742fb49
+  data.tar.gz: 1879b26b2b96ce4083a66f41d15abf07ab61eb2d264308076a65a79adb319df2
 SHA512:
-  metadata.gz: e35ca617ba6e37ad7650f14ef46fd225ce3469faac29cb91909cc682f42b160b0af6af522906cfb53015a381e8c0ddb27ebda2e1a219c186147716e38d7d9396
-  data.tar.gz: 3464e1e26a219ddb43a8a766d965608c60ef014a158ac2df923403d915a684c58fdef6f6ab75b9ccf60d1201f658c84638cb9c2e5025ac48a69b45f721121290
+  metadata.gz: 2167ced106ba8f40dc2b8eab444101b7e422722daee96f9b19cee3e596fb22ac18fd6999f8afc9f454e496fffa3fd403cdfefca4d36d32d4601148745b507832
+  data.tar.gz: 440498c252008895b164cb59214ef3f98cf4d31572694df985f62a5393c66abd71c8dcb56a57ddb4cff346b227cff1e4d9f8d18c79c375996de53efaa0031ba0

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.5.3 (2022-05-26)
+
+- Fixed bug in results with MySQL and multiple goals
+
+## 0.5.2 (2021-12-16)
+
+- Fixed error with `events` association
+- Fixed `exclude` option with custom logic
+- Fixed installation error on macOS 12
+
+## 0.5.1 (2021-09-22)
+
+- Improved performance of Bayesian calculations
+
 ## 0.5.0 (2021-09-21)
 
 - Significantly improved performance of Bayesian calculations

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2016-2021 Andrew Kane
+Copyright (c) 2016-2022 Andrew Kane
 
 MIT License
 

data/README.md

@@ -34,8 +34,6 @@ mount FieldTest::Engine, at: "field_test"
 
 Be sure to [secure the dashboard](#dashboard-security) in production.
 
-![Screenshot](https://ankane.github.io/field_test/screenshot6.png)
-
 ## Getting Started
 
 Add an experiment to `config/field_test.yml`.
@@ -427,5 +425,6 @@ To get started with development:
 git clone https://github.com/ankane/field_test.git
 cd field_test
 bundle install
+bundle exec rake compile
 bundle exec rake test
 ```

data/app/models/field_test/membership.rb

@@ -2,7 +2,7 @@ module FieldTest
   class Membership < ActiveRecord::Base
     self.table_name = "field_test_memberships"
 
-    has_many :events, class_name: "FieldTest::Event"
+    has_many :events, class_name: "FieldTest::Event", foreign_key: "field_test_membership_id"
 
     validates :participant, presence: true, if: -> { FieldTest.legacy_participants }
     validates :participant_id, presence: true, if: -> { !FieldTest.legacy_participants }

data/ext/field_test/bayestest.hpp

@@ -0,0 +1,313 @@
+/*!
+ * BayesTest C++ v0.1.0
+ * https://github.com/ankane/bayestest-cpp
+ * MIT License
+ */
+
+#pragma once
+
+#include <cmath>
+#include <vector>
+
+namespace bayestest {
+
+double logbeta(double a, double b) {
+  return std::lgamma(a) + std::lgamma(b) - std::lgamma(a + b);
+}
+
+double prob_b_beats_a(int alpha_a, int beta_a, int alpha_b, int beta_b) {
+  double total = 0.0;
+  double logbeta_aa_ba = logbeta(alpha_a, beta_a);
+  double beta_ba = beta_b + beta_a;
+
+  for (auto i = 0; i < alpha_b; i++) {
+    total += exp(logbeta(alpha_a + i, beta_ba) - log(beta_b + i) - logbeta(1 + i, beta_b) - logbeta_aa_ba);
+  }
+
+  return total;
+}
+
+double prob_c_beats_ab(int alpha_a, int beta_a, int alpha_b, int beta_b, int alpha_c, int beta_c) {
+  double total = 0.0;
+
+  double logbeta_ac_bc = logbeta(alpha_c, beta_c);
+
+  std::vector<double> log_bb_j_logbeta_j_bb;
+  log_bb_j_logbeta_j_bb.reserve(alpha_b);
+
+  for (auto j = 0; j < alpha_b; j++) {
+    log_bb_j_logbeta_j_bb.push_back(log(beta_b + j) + logbeta(1 + j, beta_b));
+  }
+
+  double abc = beta_a + beta_b + beta_c;
+  std::vector<double> logbeta_ac_i_j;
+  logbeta_ac_i_j.reserve(alpha_a + alpha_b);
+
+  for (auto i = 0; i < alpha_a + alpha_b; i++) {
+    logbeta_ac_i_j.push_back(logbeta(alpha_c + i, abc));
+  }
+
+  for (auto i = 0; i < alpha_a; i++) {
+    double sum_i = -log(beta_a + i) - logbeta(1 + i, beta_a) - logbeta_ac_bc;
+
+    for (auto j = 0; j < alpha_b; j++) {
+      total += exp(sum_i + logbeta_ac_i_j[i + j] - log_bb_j_logbeta_j_bb[j]);
+    }
+  }
+
+  return 1 - prob_b_beats_a(alpha_c, beta_c, alpha_a, beta_a) -
+    prob_b_beats_a(alpha_c, beta_c, alpha_b, beta_b) + total;
+}
+
+double prob_d_beats_abc(int alpha_a, int beta_a, int alpha_b, int beta_b, int alpha_c, int beta_c, int alpha_d, int beta_d) {
+  double total = 0.0;
+
+  double logbeta_ad_bd = logbeta(alpha_d, beta_d);
+
+  std::vector<double> log_bb_j_logbeta_j_bb;
+  log_bb_j_logbeta_j_bb.reserve(alpha_b);
+
+  for (auto j = 0; j < alpha_b; j++) {
+    log_bb_j_logbeta_j_bb.push_back(log(beta_b + j) + logbeta(1 + j, beta_b));
+  }
+
+  std::vector<double> log_bc_k_logbeta_k_bc;
+  log_bc_k_logbeta_k_bc.reserve(alpha_c);
+
+  for (auto k = 0; k < alpha_c; k++) {
+    log_bc_k_logbeta_k_bc.push_back(log(beta_c + k) + logbeta(1 + k, beta_c));
+  }
+
+  double abcd = beta_a + beta_b + beta_c + beta_d;
+  std::vector<double> logbeta_bd_i_j_k;
+  logbeta_bd_i_j_k.reserve(alpha_a + alpha_b + alpha_c);
+
+  for (auto i = 0; i < alpha_a + alpha_b + alpha_c; i++) {
+    logbeta_bd_i_j_k.push_back(logbeta(alpha_d + i, abcd));
+  }
+
+  for (auto i = 0; i < alpha_a; i++) {
+    double sum_i = -log(beta_a + i) - logbeta(1 + i, beta_a) - logbeta_ad_bd;
+
+    for (auto j = 0; j < alpha_b; j++) {
+      double sum_j = sum_i - log_bb_j_logbeta_j_bb[j];
+
+      for (auto k = 0; k < alpha_c; k++) {
+        total += exp(sum_j + logbeta_bd_i_j_k[i + j + k] - log_bc_k_logbeta_k_bc[k]);
+      }
+    }
+  }
+
+  return 1 - prob_b_beats_a(alpha_a, beta_a, alpha_d, beta_d) -
+    prob_b_beats_a(alpha_b, beta_b, alpha_d, beta_d) -
+    prob_b_beats_a(alpha_c, beta_c, alpha_d, beta_d) +
+    prob_c_beats_ab(alpha_a, beta_a, alpha_b, beta_b, alpha_d, beta_d) +
+    prob_c_beats_ab(alpha_a, beta_a, alpha_c, beta_c, alpha_d, beta_d) +
+    prob_c_beats_ab(alpha_b, beta_b, alpha_c, beta_c, alpha_d, beta_d) - total;
+}
+
+double prob_1_beats_2(int alpha_1, int beta_1, int alpha_2, int beta_2) {
+  double total = 0.0;
+  double log_b1 = log(beta_1);
+  double a2_log_b2 = alpha_2 * log(beta_2);
+  double log_b1_b2 = log(beta_1 + beta_2);
+
+  for (auto k = 0; k < alpha_1; k++) {
+    total += exp(k * log_b1 +
+      a2_log_b2 -
+      (k + alpha_2) * log_b1_b2 -
+      log(k + alpha_2) -
+      logbeta(k + 1, alpha_2));
+  }
+
+  return total;
+}
+
+double prob_1_beats_23(int alpha_1, int beta_1, int alpha_2, int beta_2, int alpha_3, int beta_3) {
+  double total = 0.0;
+
+  double log_b1_b2_b3 = log(beta_1 + beta_2 + beta_3);
+  double a1_log_b1 = alpha_1 * log(beta_1);
+  double log_b2 = log(beta_2);
+  double log_b3 = log(beta_3);
+  double loggamma_a1 = std::lgamma(alpha_1);
+
+  for (auto k = 0; k < alpha_2; k++) {
+    double sum_k = a1_log_b1 + k * log_b2 - std::lgamma(k + 1);
+
+    for (auto l = 0; l < alpha_3; l++) {
+      total += exp(sum_k + l * log_b3
+        - (k + l + alpha_1) * log_b1_b2_b3
+        + std::lgamma(k + l + alpha_1) - std::lgamma(l + 1) - loggamma_a1);
+    }
+  }
+
+  return 1.0 - prob_1_beats_2(alpha_2, beta_2, alpha_1, beta_1)
+    - prob_1_beats_2(alpha_3, beta_3, alpha_1, beta_1) + total;
+}
+
+class BinaryTest {
+public:
+  void add(int participants, int conversions) {
+    variants.emplace_back(participants, conversions);
+  }
+
+  std::vector<double> probabilities() {
+    std::vector<double> probs;
+    probs.reserve(variants.size());
+
+    switch (variants.size()) {
+      case 0: {
+        break;
+      }
+      case 1: {
+        probs.push_back(1);
+
+        break;
+      }
+      case 2: {
+        auto b = variants[0];
+        auto a = variants[1];
+
+        auto prob = prob_b_beats_a(
+          1 + a.conversions,
+          1 + a.participants - a.conversions,
+          1 + b.conversions,
+          1 + b.participants - b.conversions
+        );
+        probs.push_back(prob);
+        probs.push_back(1 - prob);
+
+        break;
+      }
+      case 3: {
+        auto total = 0.0;
+        for (auto i = 0; i < 2; i++) {
+            auto c = variants[i];
+            auto b = variants[(i + 1) % 3];
+            auto a = variants[(i + 2) % 3];
+
+            auto prob = prob_c_beats_ab(
+              1 + a.conversions,
+              1 + a.participants - a.conversions,
+              1 + b.conversions,
+              1 + b.participants - b.conversions,
+              1 + c.conversions,
+              1 + c.participants - c.conversions
+            );
+
+            probs.push_back(prob);
+            total += prob;
+        }
+        probs.push_back(1 - total);
+
+        break;
+      }
+      default: {
+        auto total = 0.0;
+        for (auto i = 0; i < 3; i++) {
+            auto d = variants[i];
+            auto c = variants[(i + 1) % 4];
+            auto b = variants[(i + 2) % 4];
+            auto a = variants[(i + 3) % 4];
+
+            auto prob = prob_d_beats_abc(
+              1 + a.conversions,
+              1 + a.participants - a.conversions,
+              1 + b.conversions,
+              1 + b.participants - b.conversions,
+              1 + c.conversions,
+              1 + c.participants - c.conversions,
+              1 + d.conversions,
+              1 + d.participants - d.conversions
+            );
+
+            probs.push_back(prob);
+            total += prob;
+        }
+        probs.push_back(1 - total);
+      }
+    }
+    return probs;
+  }
+
+private:
+  struct Variant {
+    Variant(int participants, int conversions) : participants(participants), conversions(conversions) {};
+    int participants;
+    int conversions;
+  };
+
+  std::vector<Variant> variants;
+};
+
+class CountTest {
+public:
+  void add(int events, int exposure) {
+    variants.emplace_back(events, exposure);
+  }
+
+  std::vector<double> probabilities() {
+    std::vector<double> probs;
+    probs.reserve(variants.size());
+
+    switch (variants.size()) {
+      case 0: {
+        break;
+      }
+      case 1: {
+        probs.push_back(1);
+
+        break;
+      }
+      case 2: {
+        auto a = variants[0];
+        auto b = variants[1];
+
+        auto prob = prob_1_beats_2(
+          a.events,
+          a.exposure,
+          b.events,
+          b.exposure
+        );
+        probs.push_back(prob);
+        probs.push_back(1 - prob);
+
+        break;
+      }
+      default: {
+        auto total = 0.0;
+        for (auto i = 0; i < 2; i++) {
+            auto a = variants[i];
+            auto b = variants[(i + 1) % 3];
+            auto c = variants[(i + 2) % 3];
+
+            auto prob = prob_1_beats_23(
+              a.events,
+              a.exposure,
+              b.events,
+              b.exposure,
+              c.events,
+              c.exposure
+            );
+
+            probs.push_back(prob);
+            total += prob;
+        }
+        probs.push_back(1 - total);
+      }
+    }
+    return probs;
+  }
+
+private:
+  struct Variant {
+    Variant(int events, int exposure) : events(events), exposure(exposure) {};
+    int events;
+    int exposure;
+  };
+
+  std::vector<Variant> variants;
+};
+
+}

data/ext/field_test/ext.cpp

@@ -1,12 +1,16 @@
 #include <rice/rice.hpp>
-#include "bayesian_ab.hpp"
+#include <rice/stl.hpp>
+
+#include "bayestest.hpp"
+
+using bayestest::BinaryTest;
 
 extern "C"
 void Init_ext() {
   auto rb_mFieldTest = Rice::define_module("FieldTest");
 
-  Rice::define_class_under(rb_mFieldTest, "Calculations")
-    .define_singleton_function("prob_b_beats_a", &bayesian_ab::prob_b_beats_a)
-    .define_singleton_function("prob_c_beats_a_and_b", &bayesian_ab::prob_c_beats_a_and_b)
-    .define_singleton_function("prob_d_beats_a_and_b_and_c", &bayesian_ab::prob_d_beats_a_and_b_and_c);
+  Rice::define_class_under<BinaryTest>(rb_mFieldTest, "BinaryTest")
+    .define_constructor(Rice::Constructor<BinaryTest>())
+    .define_method("add", &BinaryTest::add)
+    .define_method("probabilities", &BinaryTest::probabilities);
 }

data/lib/field_test/experiment.rb

@@ -113,10 +113,9 @@ module FieldTest
             :participant
           elsif adapter_name =~ /postg/i # postgres
             "(participant_type, participant_id)"
-          elsif adapter_name =~ /mysql/i
-            "participant_type, participant_id"
           else
-            # not perfect, but it'll do
+            # SQLite supports single column
+            # MySQL supports multiple columns, but none can be NULL
             "COALESCE(participant_type, '') || ':' || participant_id"
           end
 
@@ -141,40 +140,21 @@ module FieldTest
         }
       end
 
-      case variants.size
-      when 1, 2, 3
-        total = 0.0
-
-        (variants.size - 1).times do |i|
-          c = results.values[i]
-          b = results.values[(i + 1) % variants.size]
-          a = results.values[(i + 2) % variants.size]
-
-          alpha_a = 1 + a[:converted]
-          beta_a = 1 + a[:participated] - a[:converted]
-          alpha_b = 1 + b[:converted]
-          beta_b = 1 + b[:participated] - b[:converted]
-          alpha_c = 1 + c[:converted]
-          beta_c = 1 + c[:participated] - c[:converted]
-
-          # TODO calculate this incrementally by caching intermediate results
-          prob_winning =
-            if variants.size == 2
-              cache_fetch ["field_test", "prob_b_beats_a", alpha_b, beta_b, alpha_c, beta_c] do
-                Calculations.prob_b_beats_a(alpha_b, beta_b, alpha_c, beta_c)
-              end
-            else
-              cache_fetch ["field_test", "prob_c_beats_a_and_b", alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c] do
-                Calculations.prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_c, beta_c)
-              end
+      if variants.size <= 3
+        probabilities =
+          cache_fetch(["field_test", "probabilities"] + results.flat_map { |_, v| [v[:participated], v[:converted]] }) do
+            binary_test = BinaryTest.new
+            results.each do |_, v|
+              binary_test.add(v[:participated], v[:converted])
             end
+            binary_test.probabilities.to_a
+          end
 
-          results[variants[i]][:prob_winning] = prob_winning
-          total += prob_winning
+        results.each_key.zip(probabilities) do |variant, prob_winning|
+          results[variant][:prob_winning] = prob_winning
         end
-
-        results[variants.last][:prob_winning] = 1 - total
       end
+
       results
     end
 

data/lib/field_test/helpers.rb

@@ -13,7 +13,7 @@ module FieldTest
         end
 
         if FieldTest.exclude_bots?
-          options[:exclude] = Browser.new(request.user_agent).bot?
+          options[:exclude] ||= Browser.new(request.user_agent).bot?
         end
 
         options[:exclude] ||= FieldTest.excluded_ips.any? { |ip| ip.include?(request.remote_ip) }

data/lib/field_test/version.rb

@@ -1,3 +1,3 @@
 module FieldTest
-  VERSION = "0.5.0"
+  VERSION = "0.5.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: field_test
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-22 00:00:00.000000000 Z
+date: 2022-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -89,7 +89,7 @@ files:
 - app/views/field_test/participants/show.html.erb
 - app/views/layouts/field_test/application.html.erb
 - config/routes.rb
-- ext/field_test/bayesian_ab.hpp
+- ext/field_test/bayestest.hpp
 - ext/field_test/ext.cpp
 - ext/field_test/extconf.rb
 - lib/field_test.rb
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: A/B testing for Rails

data/ext/field_test/bayesian_ab.hpp

@@ -1,120 +0,0 @@
-#pragma once
-
-#include <cmath>
-#include <vector>
-
-namespace bayesian_ab {
-
-double logbeta(double a, double b) {
-  return std::lgamma(a) + std::lgamma(b) - std::lgamma(a + b);
-}
-
-double prob_b_beats_a(int alpha_a, int beta_a, int alpha_b, int beta_b) {
-  double total = 0.0;
-  double logbeta_aa_ba = logbeta(alpha_a, beta_a);
-  double beta_ba = beta_b + beta_a;
-
-  for (auto i = 0; i < alpha_b; i++) {
-    total += exp(logbeta(alpha_a + i, beta_ba) - log(beta_b + i) - logbeta(1 + i, beta_b) - logbeta_aa_ba);
-  }
-
-  return total;
-}
-
-double prob_c_beats_a_and_b(int alpha_a, int beta_a, int alpha_b, int beta_b, int alpha_c, int beta_c) {
-  double total = 0.0;
-
-  double logbeta_ac_bc = logbeta(alpha_c, beta_c);
-
-  std::vector<double> log_bb_j;
-  log_bb_j.reserve(alpha_b);
-  std::vector<double> logbeta_j_bb;
-  logbeta_j_bb.reserve(alpha_b);
-
-  for (auto j = 0; j < alpha_b; j++) {
-    log_bb_j.push_back(log(beta_b + j));
-    logbeta_j_bb.push_back(logbeta(1 + j, beta_b));
-  }
-
-  double abc = beta_a + beta_b + beta_c;
-  std::vector<double> logbeta_ac_i_j;
-  logbeta_ac_i_j.reserve(alpha_a + alpha_b);
-
-  for (auto i = 0; i < alpha_a + alpha_b; i++) {
-    logbeta_ac_i_j.push_back(logbeta(alpha_c + i, abc));
-  }
-
-  for (auto i = 0; i < alpha_a; i++) {
-    double sum_i = -log(beta_a + i) - logbeta(1 + i, beta_a) - logbeta_ac_bc;
-
-    for (auto j = 0; j < alpha_b; j++) {
-      total += exp(sum_i + logbeta_ac_i_j[i + j] - log_bb_j[j] - logbeta_j_bb[j]);
-    }
-  }
-
-  return 1 - prob_b_beats_a(alpha_c, beta_c, alpha_a, beta_a) -
-    prob_b_beats_a(alpha_c, beta_c, alpha_b, beta_b) + total;
-}
-
-double prob_d_beats_a_and_b_and_c(int alpha_a, int beta_a, int alpha_b, int beta_b, int alpha_c, int beta_c, int alpha_d, int beta_d) {
-  double total = 0.0;
-
-  double logbeta_ad_bd = logbeta(alpha_d, beta_d);
-
-  std::vector<double> log_bb_j;
-  log_bb_j.reserve(alpha_b);
-  std::vector<double> logbeta_j_bb;
-  logbeta_j_bb.reserve(alpha_b);
-
-  for (auto j = 0; j < alpha_b; j++) {
-    log_bb_j.push_back(log(beta_b + j));
-    logbeta_j_bb.push_back(logbeta(1 + j, beta_b));
-  }
-
-  std::vector<double> log_bc_k;
-  log_bc_k.reserve(alpha_c);
-  std::vector<double> logbeta_k_bc;
-  logbeta_k_bc.reserve(alpha_c);
-
-  for (auto k = 0; k < alpha_c; k++) {
-    log_bc_k.push_back(log(beta_c + k));
-    logbeta_k_bc.push_back(logbeta(1 + k, beta_c));
-  }
-
-  double abcd = beta_a + beta_b + beta_c + beta_d;
-  std::vector<double> logbeta_bd_i_j_k;
-  logbeta_bd_i_j_k.reserve(alpha_a + alpha_b + alpha_c);
-
-  for (auto i = 0; i < alpha_a + alpha_b + alpha_c; i++) {
-    logbeta_bd_i_j_k.push_back(logbeta(alpha_d + i, abcd));
-  }
-
-  std::vector<double> sum_i_j;
-  sum_i_j.reserve(alpha_a * alpha_b);
-  for (auto i = 0; i < alpha_a; i++) {
-    double sum_i = -log(beta_a + i) - logbeta(1 + i, beta_a) - logbeta_ad_bd;
-
-    for (auto j = 0; j < alpha_b; j++) {
-      sum_i_j.push_back(sum_i - log_bb_j[j] - logbeta_j_bb[j]);
-    }
-  }
-
-  for (auto i = 0; i < alpha_a; i++) {
-    for (auto j = 0; j < alpha_b; j++) {
-      double sum_j = sum_i_j[i * alpha_b + j];
-
-      for (auto k = 0; k < alpha_c; k++) {
-        total += exp(sum_j + logbeta_bd_i_j_k[i + j + k] - log_bc_k[k] - logbeta_k_bc[k]);
-      }
-    }
-  }
-
-  return 1 - prob_b_beats_a(alpha_a, beta_a, alpha_d, beta_d) -
-    prob_b_beats_a(alpha_b, beta_b, alpha_d, beta_d) -
-    prob_b_beats_a(alpha_c, beta_c, alpha_d, beta_d) +
-    prob_c_beats_a_and_b(alpha_a, beta_a, alpha_b, beta_b, alpha_d, beta_d) +
-    prob_c_beats_a_and_b(alpha_a, beta_a, alpha_c, beta_c, alpha_d, beta_d) +
-    prob_c_beats_a_and_b(alpha_b, beta_b, alpha_c, beta_c, alpha_d, beta_d) - total;
-}
-
-}