skiftet_statistical 0.1.0 → 0.4.0

data/lib/skiftet_statistical/significance.rb

@@ -1,12 +1,21 @@
 # frozen_string_literal: true
 
 module SkiftetStatistical
-  # Frequentist significance testing for A/B experiments. Consolidates the
-  # two-proportion z-test, Welch's t-test and the normal CDF that were previously
-  # re-implemented (inconsistently) across mej.la's AbTestAnalyzer and skram.la's
-  # CRM::AbTestAnalysis. One correct, exact (erf-based) normal CDF — no polynomial
-  # approximations.
+  # Frequentist significance testing for A/B experiments. One home for the
+  # two-proportion z-test, Welch's t-test and the normal CDF, which callers
+  # otherwise re-implement, often inconsistently. One correct, exact (erf-based)
+  # normal CDF — no polynomial approximations. Since 0.2 also the normal
+  # quantile, the chi-squared survival and the sample-ratio check.
   module Significance
+    # normal_quantile bisects [-40, 40]: 0.5 * erfc(40 / sqrt(2)) underflows to 0.0,
+    # so every p a Float can hold strictly inside (0, 1) has its z in there.
+    NORMAL_QUANTILE_LIMIT = 40.0
+    # 60 halvings leave a bracket of 80 / 2**60 (about 7e-17).
+    NORMAL_QUANTILE_STEPS = 60
+    # Under this many observations a sample-ratio check is not run: too few to tell
+    # a broken split from chance.
+    SAMPLE_RATIO_MINIMUM_TOTAL = 200
+
     module_function
 
     # Standard normal cumulative distribution Phi(z), exact via erf.
@@ -49,6 +58,57 @@ module SkiftetStatistical
       Result.new(statistic: t, p_value: two_tailed_p_value(t))
     end
 
+    # The z where the standard normal distribution function is p (the probit), by
+    # bisection on erfc. -Infinity at p = 0 and Infinity at p = 1.
+    def normal_quantile(p)
+      p = p.to_f
+      raise ArgumentError, "p must be between 0 and 1, got #{p}" unless p.between?(0.0, 1.0)
+      return -Float::INFINITY if p.zero?
+      return Float::INFINITY if p == 1.0
+
+      low = -NORMAL_QUANTILE_LIMIT
+      high = NORMAL_QUANTILE_LIMIT
+      NORMAL_QUANTILE_STEPS.times do
+        middle = (low + high) / 2
+        0.5 * Math.erfc(-middle / Math.sqrt(2.0)) < p ? low = middle : high = middle
+      end
+      (low + high) / 2
+    end
+
+    # P(X >= statistic) for X ~ chi-squared(degrees_of_freedom): the regularized
+    # upper incomplete gamma Q(dof / 2, statistic / 2). 1.0 for a statistic <= 0.
+    def chi_squared_survival(statistic, degrees_of_freedom)
+      return 1.0 if statistic <= 0.0
+
+      Gamma.regularized_upper(degrees_of_freedom / 2.0, statistic / 2.0)
+    end
+
+    # The sample-ratio mismatch check: does each arm's observed count fit the count
+    # its share of the traffic leads us to expect? Pearson's chi-squared over the
+    # arms expected to get traffic, on k - 1 degrees of freedom. A small p means the
+    # split does not do what its weights say (a broken arm, a cache, a bot).
+    #
+    #   Significance.sample_ratio_p_value(observed: { control: 700, a: 300 },
+    #                                     expected: { control: 500.0, a: 500.0 }) # => ~1.1e-36
+    #
+    # The arms are observed's keys (expected is read by the same keys, missing = 0).
+    # nil (not tested) under minimum_total observations or with fewer than two arms
+    # expected to get traffic; 0.0 when an arm expected to get nothing got some,
+    # a mismatch no count explains.
+    def sample_ratio_p_value(observed:, expected:, minimum_total: SAMPLE_RATIO_MINIMUM_TOTAL)
+      total = observed.values.sum
+      return nil if total < minimum_total
+
+      cells = observed.keys.map { |arm| [ observed.fetch(arm).to_f, expected.fetch(arm, 0.0).to_f ] }
+      return 0.0 if cells.any? { |seen, wanted| wanted <= 0.0 && seen.positive? }
+
+      cells = cells.select { |_seen, wanted| wanted.positive? }
+      return nil if cells.size < 2
+
+      statistic = cells.sum { |seen, wanted| ((seen - wanted)**2) / wanted }
+      chi_squared_survival(statistic, cells.size - 1)
+    end
+
     # The outcome of a significance test: the test statistic and its two-tailed
     # p-value, with convenience predicates for the usual confidence levels.
     Result = Struct.new(:statistic, :p_value, keyword_init: true) do

data/lib/skiftet_statistical/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SkiftetStatistical
-  VERSION = "0.1.0"
+  VERSION = "0.4.0"
 end

data/lib/skiftet_statistical.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
 require_relative "skiftet_statistical/version"
+require_relative "skiftet_statistical/numbers"
 require_relative "skiftet_statistical/sampler"
+require_relative "skiftet_statistical/gamma"
+require_relative "skiftet_statistical/beta_function"
+require_relative "skiftet_statistical/joint_beta"
 require_relative "skiftet_statistical/descriptive"
 require_relative "skiftet_statistical/significance"
 require_relative "skiftet_statistical/arm"
@@ -11,22 +15,70 @@ require_relative "skiftet_statistical/policies/thompson_sampling"
 require_relative "skiftet_statistical/policies/ucb1"
 require_relative "skiftet_statistical/policies/softmax"
 require_relative "skiftet_statistical/bandit"
+require_relative "skiftet_statistical/bandit/checks"
+require_relative "skiftet_statistical/bandit/tallies"
+require_relative "skiftet_statistical/bandit/prior"
+require_relative "skiftet_statistical/bandit/decay"
+require_relative "skiftet_statistical/bandit/draw"
+require_relative "skiftet_statistical/bandit/normal_argmax"
+require_relative "skiftet_statistical/bandit/posteriors"
+require_relative "skiftet_statistical/bandit/ranking"
+require_relative "skiftet_statistical/bandit/summary"
+require_relative "skiftet_statistical/bandit/probabilities"
+require_relative "skiftet_statistical/bandit/pass"
+require_relative "skiftet_statistical/bandit/weighted_posterior"
+require_relative "skiftet_statistical/bandit/allocator"
+require_relative "skiftet_statistical/bandit/win_hold"
+require_relative "skiftet_statistical/cholesky"
+require_relative "skiftet_statistical/linear_value"
+require_relative "skiftet_statistical/hierarchical_rates"
+require_relative "skiftet_statistical/shared_credit"
+require_relative "skiftet_statistical/gauss_legendre"
+require_relative "skiftet_statistical/beta_argmax"
+require_relative "skiftet_statistical/additive_contrasts"
+require_relative "skiftet_statistical/prequential"
+require_relative "skiftet_statistical/cusum"
 
-# Skiftet's shared statistics toolkit — a home for reusable, app-independent
-# statistical analysis code across the workspace.
+# A dependency-free statistics toolkit: reusable, app-independent statistical
+# analysis code, so the same test, interval or draw is written once.
 #
 # Modules:
 # - {Descriptive} — mean, variance, standard deviation, percentiles/median.
 # - {Significance} — A/B significance testing (two-proportion z-test, Welch's
-#   t-test, normal CDF / two-tailed p-values).
+#   t-test, normal CDF / two-tailed p-values, normal quantile, chi-squared
+#   survival, the sample-ratio check).
+# - {Gamma} — the regularized incomplete gamma functions P(a, x) and Q(a, x).
+# - {BetaFunction} — the regularized incomplete beta function and its quantile.
 # - {Sampler} — Gamma/Beta/Gaussian random sampling (RNG-injectable).
+# - {JointBeta} — joint Monte Carlo draws over Beta posteriors (P(best), loss).
 # - {Bandit} + {Policies} — multi-armed bandit (Thompson Sampling, epsilon-greedy,
-#   UCB1, Softmax) for online explore/exploit decisions.
+#   UCB1, Softmax) for online explore/exploit decisions, kept as 0.3.0 wrote it.
+# - {Bandit::Posteriors} — the generic bandit: a set of items with their views
+#   and yes/no or amount posteriors, ordered (`rank`) and picked (`pick`) by one
+#   rule from a key, given uniforms or an RNG, with the odds of what it draws
+#   (exact for a pick among up to EXACT_PICK_MAX contenders and for orders and
+#   positions over up to EXACT_ORDER_MAX free items with none barred from
+#   leading, otherwise a seeded Monte Carlo estimate with its standard error),
+#   and per-item summaries; its priors
+#   ({Bandit::Prior}), decay ({Bandit::Decay}) and keyed uniforms and draw by
+#   weight ({Bandit::Draw}) beside it, with {Bandit::NormalArgmax} and
+#   {Bandit::Pass}.
+# - {Bandit::Allocator}, {Bandit::WinHold}, {Bandit::WeightedPosterior} — an A/B
+#   test's Thompson-sampling allocation with a floor, a step limit, an automatic
+#   win and a hold on it.
+# - {LinearValue}, {Cholesky}, {SharedCredit}, {HierarchicalRates},
+#   {AdditiveContrasts}, {BetaArgmax}, {GaussLegendre}, {Prequential}, {Cusum} —
+#   values of several parts, pooled estimates, the exact Beta argmax, the
+#   prequential score and a change test.
 #
-#   bandit = SkiftetStatistical.bandit(arms: %w[facebook whatsapp x])
+#   bandit = SkiftetStatistical.bandit(arms: %w[a b c])
 #   choice = bandit.select          # which arm to play now
 #   bandit.record(choice, 1)        # observed a reward (e.g. a conversion)
 #   bandit.best_arm                 # current best by empirical mean
+#
+#   set = SkiftetStatistical::Bandit::Posteriors.yes_no({ "a" => { views: 40, outcomes: 8 },
+#                                                       "b" => { views: 10, outcomes: 1 } })
+#   set.rank(key: %w[list visit-1], shuffle: 0.1).items  # => the order that key draws, every time
 module SkiftetStatistical
   class Error < StandardError; end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skiftet_statistical
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Skiftet
@@ -10,13 +10,18 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: |-
-  A small, dependency-free toolkit for online decision-making under uncertainty:
-  register arms, ask which to play, record rewards, and let a pluggable policy
-  balance exploration and exploitation. Ships Thompson Sampling, epsilon-greedy,
-  UCB1 and Softmax; state serialises to a plain Hash for persistence and every
-  policy is deterministic under an injected RNG for testing.
-email:
-- joel@skram.la
+  Pure Ruby statistics for deciding under uncertainty. Descriptive statistics,
+  significance tests and distribution functions; Beta and Normal posteriors with
+  exact P(best), expected loss and credible intervals; an A/B allocator with
+  floors, fixed shares, an automatic win and a hold on it; estimators for values
+  of several parts, pooled contrasts, hierarchical rates and a change test; and
+  a generic Thompson-sampling bandit: a set of items with their views and
+  yes/no or amount outcomes, ranked or picked by one rule from a key, given
+  uniforms or an RNG, with the odds of what it draws: exact for a pick among
+  up to 20 contenders (EXACT_PICK_MAX) and for orders and positions over up to
+  10 free items (EXACT_ORDER_MAX) with none barred from leading, otherwise a
+  seeded Monte Carlo estimate with its standard error. Every random input is
+  injected, so each answer can be reproduced.
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -25,15 +30,43 @@ files:
 - LICENSE.txt
 - README.md
 - lib/skiftet_statistical.rb
+- lib/skiftet_statistical/additive_contrasts.rb
 - lib/skiftet_statistical/arm.rb
 - lib/skiftet_statistical/bandit.rb
+- lib/skiftet_statistical/bandit/allocator.rb
+- lib/skiftet_statistical/bandit/checks.rb
+- lib/skiftet_statistical/bandit/decay.rb
+- lib/skiftet_statistical/bandit/draw.rb
+- lib/skiftet_statistical/bandit/normal_argmax.rb
+- lib/skiftet_statistical/bandit/pass.rb
+- lib/skiftet_statistical/bandit/posteriors.rb
+- lib/skiftet_statistical/bandit/prior.rb
+- lib/skiftet_statistical/bandit/probabilities.rb
+- lib/skiftet_statistical/bandit/ranking.rb
+- lib/skiftet_statistical/bandit/summary.rb
+- lib/skiftet_statistical/bandit/tallies.rb
+- lib/skiftet_statistical/bandit/weighted_posterior.rb
+- lib/skiftet_statistical/bandit/win_hold.rb
+- lib/skiftet_statistical/beta_argmax.rb
+- lib/skiftet_statistical/beta_function.rb
+- lib/skiftet_statistical/cholesky.rb
+- lib/skiftet_statistical/cusum.rb
 - lib/skiftet_statistical/descriptive.rb
+- lib/skiftet_statistical/gamma.rb
+- lib/skiftet_statistical/gauss_legendre.rb
+- lib/skiftet_statistical/hierarchical_rates.rb
+- lib/skiftet_statistical/inputs.rb
+- lib/skiftet_statistical/joint_beta.rb
+- lib/skiftet_statistical/linear_value.rb
+- lib/skiftet_statistical/numbers.rb
 - lib/skiftet_statistical/policies/base.rb
 - lib/skiftet_statistical/policies/epsilon_greedy.rb
 - lib/skiftet_statistical/policies/softmax.rb
 - lib/skiftet_statistical/policies/thompson_sampling.rb
 - lib/skiftet_statistical/policies/ucb1.rb
+- lib/skiftet_statistical/prequential.rb
 - lib/skiftet_statistical/sampler.rb
+- lib/skiftet_statistical/shared_credit.rb
 - lib/skiftet_statistical/significance.rb
 - lib/skiftet_statistical/version.rb
 homepage: https://github.com/Skiftet/skiftet_statistical
@@ -42,7 +75,6 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   github_repo: https://github.com/Skiftet/skiftet_statistical
-  homepage_uri: https://github.com/Skiftet/skiftet_statistical
   source_code_uri: https://github.com/Skiftet/skiftet_statistical
   changelog_uri: https://github.com/Skiftet/skiftet_statistical/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
@@ -62,6 +94,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.9
 specification_version: 4
-summary: Multi-armed bandit policies (Thompson Sampling, epsilon-greedy, UCB1, Softmax)
-  for Ruby.
+summary: 'A dependency-free statistics toolkit for Ruby: significance tests, Bayesian
+  A/B allocation and a Thompson-sampling bandit that picks and orders items.'
 test_files: []