skiftet_statistical 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +308 -8
  3. data/README.md +1416 -39
  4. data/lib/skiftet_statistical/additive_contrasts.rb +894 -0
  5. data/lib/skiftet_statistical/bandit/allocator.rb +477 -63
  6. data/lib/skiftet_statistical/bandit/checks.rb +128 -0
  7. data/lib/skiftet_statistical/bandit/decay.rb +86 -0
  8. data/lib/skiftet_statistical/bandit/draw.rb +179 -0
  9. data/lib/skiftet_statistical/bandit/normal_argmax.rb +209 -0
  10. data/lib/skiftet_statistical/bandit/pass.rb +94 -0
  11. data/lib/skiftet_statistical/bandit/posteriors.rb +497 -0
  12. data/lib/skiftet_statistical/bandit/prior.rb +264 -0
  13. data/lib/skiftet_statistical/bandit/probabilities.rb +929 -0
  14. data/lib/skiftet_statistical/bandit/ranking.rb +552 -0
  15. data/lib/skiftet_statistical/bandit/summary.rb +152 -0
  16. data/lib/skiftet_statistical/bandit/tallies.rb +297 -0
  17. data/lib/skiftet_statistical/bandit.rb +3 -3
  18. data/lib/skiftet_statistical/beta_argmax.rb +526 -0
  19. data/lib/skiftet_statistical/beta_function.rb +28 -1
  20. data/lib/skiftet_statistical/cholesky.rb +244 -0
  21. data/lib/skiftet_statistical/cusum.rb +183 -0
  22. data/lib/skiftet_statistical/descriptive.rb +3 -3
  23. data/lib/skiftet_statistical/gauss_legendre.rb +73 -0
  24. data/lib/skiftet_statistical/hierarchical_rates.rb +450 -0
  25. data/lib/skiftet_statistical/inputs.rb +214 -0
  26. data/lib/skiftet_statistical/linear_value.rb +364 -0
  27. data/lib/skiftet_statistical/numbers.rb +98 -0
  28. data/lib/skiftet_statistical/prequential.rb +206 -0
  29. data/lib/skiftet_statistical/shared_credit.rb +811 -0
  30. data/lib/skiftet_statistical/significance.rb +5 -6
  31. data/lib/skiftet_statistical/version.rb +1 -1
  32. data/lib/skiftet_statistical.rb +43 -4
  33. metadata +37 -11
@@ -0,0 +1,450 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SkiftetStatistical
4
+ # Hierarchical Poisson rates: how far a node's rate stands from its parent's, from the node's count s and the count E
5
+ # its parent leads us to expect there. The node's ratio theta = rate / expected rate has a Gamma prior with mean 1 and
6
+ # strength k (Clayton and Kaldor 1987), so its posterior is Gamma(k + s, k + E). Under a two-groups prior (Efron 2008)
7
+ # the node is null with probability 1 - h (its count Poisson(E), its ratio 1) and else drawn from that Gamma slab, so
8
+ # its count is negative binomial NB(s; k, k / (k + E)). Beside the node rule: the leave-one-out share that sets E, the
9
+ # level fit of k (and h) by grid marginal maximum likelihood, the leaf fit that estimates each parent's null rate
10
+ # jointly with k and h, the smoothing of a fitted value across runs, Marshall's (1991) moment estimate and the
11
+ # boundary likelihood-ratio test (Self and Liang 1987).
12
+ #
13
+ # Counts are real numbers >= 0 throughout, so a caller may count by share: a node that stands in its parent only by a
14
+ # share w has w multiplied into its count and total by the caller (there is no weight keyword). The probability mass
15
+ # functions are taken in logs and through log-gamma, s! being Gamma(s + 1), with no rounding.
16
+ #
17
+ # SkiftetStatistical::HierarchicalRates.node(count: 6.0, expected: 2.5, strength: 8.0, threshold: 1.25,
18
+ # parent_tail: 0.0, parent_ratio: 1.0)
19
+ # # => { ratio: 1.3333333333333333 (14 / 10.5), hot: 1.0, tail: 0.5593048238168208 (Q(14, 13.125)) }
20
+ #
21
+ # The arithmetic and its order are part of the contract: an app pins these outputs at 1e-9 relative (they go through
22
+ # Math.lgamma), so a reordered formula is a breaking change even when it is "the same".
23
+ module HierarchicalRates
24
+ # The strengths a level is fitted on: 2**-2 to 2**12 in quarter powers of 2, 57 points.
25
+ GRID = Array.new(57) { |i| 2.0**(-2 + (i / 4.0)) }.freeze
26
+ # A strength's log-likelihood within this of the maximum lies in the 95 % profile interval: half the 95 % point
27
+ # of chi-squared(1).
28
+ PROFILE_DROP = 1.92
29
+ # A pair whose parent has fewer children with data than this carries nothing about its level: the fit and both
30
+ # checks leave it out.
31
+ MINIMUM_SIBLINGS = 2
32
+ # The leaf fit takes at most this many steps on a parent's null rate ...
33
+ LEAF_STEPS = 50
34
+ # ... and stops at the first step that moves the rate by at most this share of itself.
35
+ LEAF_CHANGE = 1e-10
36
+
37
+ class << self
38
+ # The parent's share with the node's own count and total taken out, smoothed toward a prior share:
39
+ # (parent_count - node_count + prior_strength * prior_share) / (parent_total - node_total + prior_strength).
40
+ def leave_one_out_share(parent_count:, parent_total:, node_count:, node_total:, prior_share:, prior_strength:)
41
+ parent_count = nonnegative(parent_count, "parent_count")
42
+ parent_total = nonnegative(parent_total, "parent_total")
43
+ node_count = nonnegative(node_count, "node_count")
44
+ node_total = nonnegative(node_total, "node_total")
45
+ prior_share = share(prior_share, "prior_share")
46
+ prior_strength = nonnegative(prior_strength, "prior_strength")
47
+
48
+ denominator = parent_total - node_total + prior_strength
49
+ unless denominator.positive?
50
+ raise ArgumentError, "parent_total - node_total + prior_strength must be > 0, got #{denominator}"
51
+ end
52
+
53
+ (parent_count - node_count + (prior_strength * prior_share)) / denominator
54
+ end
55
+
56
+ # One node: count s, expected count E, strength k, hot share h, a threshold L on the ratio to the reference
57
+ # (theta = parent_ratio * node ratio), and the parent's own P(theta > L) and ratio.
58
+ # hot = h * NB(s; k, k / (k + E)) / [h * NB + (1 - h) * Poisson(s; E)], 1.0 at h = 1
59
+ # ratio = (1 - hot) + hot * (k + s) / (k + E)
60
+ # tail = (1 - hot) * parent_tail + hot * Q(k + s, (k + E) * L / parent_ratio), Q = Gamma.regularized_upper
61
+ # A node with nothing expected (E = 0) may only have a count of 0, and then answers its prior.
62
+ def node(count:, expected:, strength:, hot_share: 1.0, threshold:, parent_tail:, parent_ratio:)
63
+ s = nonnegative(count, "count")
64
+ e = nonnegative(expected, "expected")
65
+ k = positive(strength, "strength")
66
+ h = share(hot_share, "hot_share")
67
+ limit = nonnegative(threshold, "threshold")
68
+ parent_tail = share(parent_tail, "parent_tail")
69
+ parent_ratio = positive(parent_ratio, "parent_ratio")
70
+ observable(s, e)
71
+
72
+ hot = hot_probability(s, e, k, h)
73
+ ratio = (1.0 - hot) + (hot * (k + s) / (k + e))
74
+ tail = ((1.0 - hot) * parent_tail) + (hot * Gamma.regularized_upper(k + s, (k + e) * limit / parent_ratio))
75
+ { ratio: ratio, hot: hot, tail: tail }
76
+ end
77
+
78
+ # The level fit: the marginal log-likelihood sum log[h * NB(s; k, k / (k + E)) + (1 - h) * Poisson(s; E)] over
79
+ # pairs [{ count:, expected:, siblings: }] (siblings: the children of the pair's parent that hold data, the pair's
80
+ # own node included), maximised over GRID and over hot_grid (h = 1 without one). A tie goes to the larger
81
+ # strength (the nodes then differ less), then to the hot share listed first. interval holds the grid strengths
82
+ # within PROFILE_DROP of the maximum at the fitted h; profile is [strength, log-likelihood] at every grid point
83
+ # at the fitted h. at_upper_bound: the strength is GRID's top, so the level's nodes inherit their parent.
84
+ def fit_strength(pairs:, hot_grid: nil)
85
+ level = level_terms(pairs)
86
+ hots = hot_grid.nil? ? [ 1.0 ] : hot_shares(hot_grid)
87
+
88
+ profiles = hots.map { [] }
89
+ GRID.each do |k|
90
+ terms = slab_terms(level, k)
91
+ hots.each_with_index { |h, j| profiles[j] << log_likelihood(terms, h) }
92
+ end
93
+
94
+ best_ll, best_k, best_j = best_point(profiles)
95
+ profile = profiles[best_j]
96
+ inside = GRID.each_index.select { |i| profile[i] >= best_ll - PROFILE_DROP }
97
+ {
98
+ strength: GRID[best_k],
99
+ hot_share: hots[best_j],
100
+ log_likelihood: best_ll,
101
+ at_upper_bound: best_k == GRID.size - 1,
102
+ interval: [ GRID[inside.min], GRID[inside.max] ],
103
+ profile: GRID.each_with_index.map { |k, i| [ k, profile[i] ] }
104
+ }
105
+ end
106
+
107
+ # The leaf level's two-groups fit, each parent's NULL rate estimated jointly with k and h instead of read from a
108
+ # leave-one-out share. With few leaves to a parent (about five), a hot leaf lifts its siblings' leave-one-out
109
+ # share, so they read as cold, and a fit on those expected counts reads the hot share several times too high.
110
+ #
111
+ # parents: [[{ count:, total: }]], each parent's leaves, count a real number >= 0 and total the leaf's exposure,
112
+ # > 0; a leaf expects total * rate, rate its parent's null rate. At a strength k (grid: rising strengths, GRID
113
+ # by default) and a hot share h (hot_shares: each in [0, 1), since the rate is read from the leaves that may be
114
+ # null), every parent's rate starts at its pooled rate, sum count / sum total, and each step makes it
115
+ # rate = sum w * count / sum w * total, w = the leaf's posterior probability of being null at the rate before,
116
+ # the null part of the mixture whose slab part is a node's hot. A parent stops at the first step that moves its
117
+ # rate by at most LEAF_CHANGE of itself (converged) or after LEAF_STEPS steps; parents share only k and h, so
118
+ # each is stepped on its own. A step whose sums give no rate, or a rate of 0 beside a count above 0 (which no
119
+ # rate of 0 explains), is not taken: the parent keeps its rate, not converged.
120
+ #
121
+ # The fit maximises fit_strength's marginal log-likelihood, over the leaves at their parents' rates, on grid
122
+ # times hot_shares; a tie goes to the larger strength, then to the hot share listed first. A parent with fewer
123
+ # than MINIMUM_SIBLINGS leaves is its own leaf's rate and carries nothing about the level: it is left out of
124
+ # the fit and answered at the fitted k and h. The answer, converged, rates and hot in the order given:
125
+ # { strength:, hot_share:, log_likelihood:, at_upper_bound: (the strength is grid's last), converged: [whether
126
+ # the parent's rate settled at the fitted point], rates: [rate], hot: [[posterior probability that the leaf
127
+ # is hot]] }
128
+ # A leaf's hot is node's hot at count: count, expected: total * rate, strength: and hot_share:, bit for bit. A
129
+ # parent whose one leaf is near the line between null and hot can take hundreds of steps to settle: it answers
130
+ # its rate after LEAF_STEPS steps, and converged says so.
131
+ def fit_leaves(parents:, hot_shares:, grid: GRID)
132
+ groups = leaf_groups(parents)
133
+ strengths = strength_grid(grid)
134
+ hots = leaf_hot_shares(hot_shares)
135
+ kept = groups.select { |leaves| leaves.size >= MINIMUM_SIBLINGS }
136
+ raise ArgumentError, "no parent with #{MINIMUM_SIBLINGS} or more leaves" if kept.empty?
137
+
138
+ profiles = hots.map { [] }
139
+ strengths.each do |k|
140
+ level = with_strength(kept, k)
141
+ hots.each_with_index { |h, j| profiles[j] << leaf_level(level, k, h)[:log_likelihood] }
142
+ end
143
+
144
+ best_ll, best_k, best_j = best_point(profiles)
145
+ k = strengths[best_k]
146
+ h = hots[best_j]
147
+ fitted = leaf_level(with_strength(groups, k), k, h)
148
+ {
149
+ strength: k,
150
+ hot_share: h,
151
+ log_likelihood: best_ll,
152
+ at_upper_bound: best_k == strengths.size - 1,
153
+ converged: fitted[:converged],
154
+ rates: fitted[:rates],
155
+ hot: groups.zip(fitted[:rates]).map do |leaves, rate|
156
+ leaves.map { |s, t| hot_probability(s, t * rate, k, h) }
157
+ end
158
+ }
159
+ end
160
+
161
+ # exp((1 - weight) * log previous + weight * log fitted), a moving average in logs; fitted when there is no
162
+ # previous value.
163
+ def smooth_strength(previous:, fitted:, weight: 0.2)
164
+ fitted = positive(fitted, "fitted")
165
+ return fitted if previous.nil?
166
+
167
+ previous = positive(previous, "previous")
168
+ weight = share(weight, "weight")
169
+ Math.exp(((1.0 - weight) * Math.log(previous)) + (weight * Math.log(fitted)))
170
+ end
171
+
172
+ # Marshall's moment estimate of the ratios' variance over the fit's pairs: A = sum E * (s / E - m)**2 / sum E -
173
+ # m / mean(E), m = sum s / sum E, floored at 0; strength 1 / A, infinite at 0.
174
+ def marshall(pairs:)
175
+ level = kept_pairs(pairs)
176
+ total = level.sum { |_s, e| e }
177
+ m = level.sum { |s, _e| s } / total
178
+ spread = level.sum { |s, e| e * (((s / e) - m)**2) } / total
179
+ variance = spread - (m / (total / level.size))
180
+ variance = 0.0 unless variance.positive?
181
+ { variance: variance, strength: variance.zero? ? Float::INFINITY : 1.0 / variance }
182
+ end
183
+
184
+ # Does the level matter at all? LR = 2 * (l(k, h) - l(infinite)), l the fit's log-likelihood on the fit's
185
+ # pairs and l(infinite) the all-Poisson one; p = 0.5 * chi_squared_survival(LR, 1) on the boundary, 1.0 at
186
+ # LR <= 0. The statistic is answered as computed, negative included.
187
+ def boundary_test(pairs:, strength:, hot_share: 1.0)
188
+ level = level_terms(pairs)
189
+ k = positive(strength, "strength")
190
+ h = share(hot_share, "hot_share")
191
+
192
+ statistic = 2.0 * (log_likelihood(slab_terms(level, k), h) - level.sum { |term| term[3] })
193
+ p_value = statistic > 0.0 ? 0.5 * Significance.chi_squared_survival(statistic, 1) : 1.0
194
+ { statistic: statistic, p_value: p_value }
195
+ end
196
+
197
+ private
198
+
199
+ # The posterior probability that the node is drawn from the slab: the slab's part of the same two-part mixture
200
+ # whose log is a pair's term in the level's likelihood (log_likelihood).
201
+ def hot_probability(s, e, k, h)
202
+ return 1.0 if h == 1.0
203
+ return 0.0 if h.zero?
204
+
205
+ log_hot, log_rest = log_weights(h)
206
+ log_factorial = Math.lgamma(s + 1.0)[0]
207
+ slab = log_hot + log_slab(s, e, k, Math.lgamma(k)[0], log_factorial)
208
+ null = log_rest + log_null(s, e, log_factorial)
209
+ 1.0 / (1.0 + Math.exp(null - slab))
210
+ end
211
+
212
+ # sum over pairs of log[h * NB + (1 - h) * Poisson], each term mixed in logs.
213
+ def log_likelihood(terms, h)
214
+ return terms.sum { |slab, _null| slab } if h == 1.0
215
+ return terms.sum { |_slab, null| null } if h.zero?
216
+
217
+ log_hot, log_rest = log_weights(h)
218
+ terms.sum { |slab, null| log_sum(log_hot + slab, log_rest + null) }
219
+ end
220
+
221
+ # [log h, log(1 - h)] for 0 < h < 1: the two parts' weights.
222
+ def log_weights(h)
223
+ [ Math.log(h), Math.log(1.0 - h) ]
224
+ end
225
+
226
+ # log(e**a + e**b), from the larger.
227
+ def log_sum(a, b)
228
+ if a > b
229
+ a + Math.log(1.0 + Math.exp(b - a))
230
+ else
231
+ b + Math.log(1.0 + Math.exp(a - b))
232
+ end
233
+ end
234
+
235
+ # log NB(s; k, k / (k + E)) = lgamma(s + k) - lgamma(k) - lgamma(s + 1) + k log(k / (k + E)) + s log(E / (k + E)).
236
+ def log_slab(s, e, k, log_gamma_k, log_factorial)
237
+ log_slab_from(Math.lgamma(s + k)[0], s, e, k, log_gamma_k, log_factorial)
238
+ end
239
+
240
+ # log_slab with lgamma(s + k) given, in the same order.
241
+ def log_slab_from(log_gamma_sk, s, e, k, log_gamma_k, log_factorial)
242
+ log_gamma_sk - log_gamma_k - log_factorial + (k * Math.log(k / (k + e))) + x_log_y(s, e / (k + e))
243
+ end
244
+
245
+ # log Poisson(s; E) = s log E - E - lgamma(s + 1).
246
+ def log_null(s, e, log_factorial)
247
+ x_log_y(s, e) - e - log_factorial
248
+ end
249
+
250
+ # x log y, 0 at x = 0 (a count of 0 against nothing expected).
251
+ def x_log_y(x, y)
252
+ x.zero? ? 0.0 : x * Math.log(y)
253
+ end
254
+
255
+ # [log NB, log Poisson] of every pair at strength k.
256
+ def slab_terms(level, k)
257
+ log_gamma_k = Math.lgamma(k)[0]
258
+ level.map { |s, e, log_factorial, null| [ log_slab(s, e, k, log_gamma_k, log_factorial), null ] }
259
+ end
260
+
261
+ # The kept pairs with what does not depend on k: [s, E, lgamma(s + 1), log Poisson(s; E)].
262
+ def level_terms(pairs)
263
+ kept_pairs(pairs).map do |s, e|
264
+ log_factorial = Math.lgamma(s + 1.0)[0]
265
+ [ s, e, log_factorial, log_null(s, e, log_factorial) ]
266
+ end
267
+ end
268
+
269
+ # Every pair checked, the lone ones (fewer than MINIMUM_SIBLINGS children with data) left out: [[s, E]].
270
+ def kept_pairs(pairs)
271
+ kept = pairs.filter_map do |pair|
272
+ siblings = pair.fetch(:siblings) { raise ArgumentError, "each pair needs siblings:, got #{pair.inspect}" }
273
+ unless siblings.is_a?(Integer) && siblings >= 0
274
+ raise ArgumentError, "siblings must be an Integer >= 0, got #{siblings.inspect}"
275
+ end
276
+
277
+ s = nonnegative(pair.fetch(:count) { raise ArgumentError, "each pair needs count:" }, "count")
278
+ e = positive(pair.fetch(:expected) { raise ArgumentError, "each pair needs expected:" }, "expected")
279
+ [ s, e ] if siblings >= MINIMUM_SIBLINGS
280
+ end
281
+ raise ArgumentError, "no pair whose parent has #{MINIMUM_SIBLINGS} or more children with data" if kept.empty?
282
+
283
+ kept
284
+ end
285
+
286
+ # The largest log-likelihood, ties to the larger strength, then to the hot share listed first:
287
+ # [log-likelihood, grid index, hot index].
288
+ def best_point(profiles)
289
+ best = nil
290
+ profiles.each_with_index do |profile, j|
291
+ profile.each_with_index do |ll, i|
292
+ best = [ ll, i, j ] if best.nil? || ll > best[0] || (ll == best[0] && i > best[1])
293
+ end
294
+ end
295
+ best
296
+ end
297
+
298
+ def hot_shares(hot_grid, name = "hot_grid")
299
+ hots = hot_grid.map { |h| share(h, "each #{name} value") }
300
+ raise ArgumentError, "#{name} must hold at least one hot share" if hots.empty?
301
+
302
+ hots
303
+ end
304
+
305
+ # The leaf fit's hot shares, each below 1: the null rate is read from the leaves that may be null.
306
+ def leaf_hot_shares(values)
307
+ raise ArgumentError, "hot_shares must be an Array, got #{values.inspect}" unless values.is_a?(Array)
308
+
309
+ hots = hot_shares(values, "hot_shares")
310
+ raise ArgumentError, "each hot_shares value must be below 1, got #{values.inspect}" if hots.include?(1.0)
311
+
312
+ hots
313
+ end
314
+
315
+ def strength_grid(grid)
316
+ unless grid.is_a?(Array) && !grid.empty?
317
+ raise ArgumentError, "grid must be an Array of at least one strength, got #{grid.inspect}"
318
+ end
319
+
320
+ strengths = grid.map { |k| positive(k, "each grid strength") }
321
+ raise ArgumentError, "grid must rise strictly, got #{grid.inspect}" unless strengths.each_cons(2).all? { |a, b| a < b }
322
+
323
+ strengths
324
+ end
325
+
326
+ # Every parent's leaves checked: [[[s, t, lgamma(s + 1)]]].
327
+ def leaf_groups(parents)
328
+ raise ArgumentError, "parents must be an Array of parents, got #{parents.inspect}" unless parents.is_a?(Array)
329
+
330
+ parents.map do |leaves|
331
+ unless leaves.is_a?(Array) && !leaves.empty?
332
+ raise ArgumentError, "each parent must be an Array of at least one leaf, got #{leaves.inspect}"
333
+ end
334
+
335
+ leaves.map do |leaf|
336
+ raise ArgumentError, "each leaf must be a Hash of count: and total:, got #{leaf.inspect}" unless leaf.is_a?(Hash)
337
+
338
+ s = nonnegative(leaf.fetch(:count) { raise ArgumentError, "each leaf needs count:" }, "count")
339
+ t = positive(leaf.fetch(:total) { raise ArgumentError, "each leaf needs total:" }, "total")
340
+ [ s, t, Math.lgamma(s + 1.0)[0] ]
341
+ end
342
+ end
343
+ end
344
+
345
+ # The parents at strength k, each [pooled rate, any count above 0, leaves], a leaf [s, t, lgamma(s + 1),
346
+ # lgamma(s + k), lgamma(s + k) - lgamma(k) + k log k].
347
+ def with_strength(groups, k)
348
+ base = (k * Math.log(k)) - Math.lgamma(k)[0]
349
+ groups.map do |leaves|
350
+ parent = leaves.map do |s, t, log_factorial|
351
+ log_gamma_sk = Math.lgamma(s + k)[0]
352
+ [ s, t, log_factorial, log_gamma_sk, log_gamma_sk + base ]
353
+ end
354
+ [ pooled_rate(leaves), leaves.any? { |s, _t| s.positive? }, parent ]
355
+ end
356
+ end
357
+
358
+ # Every parent's null rate at strength k and hot share h, whether it settled, and the log-likelihood of their
359
+ # leaves at those rates (fit_strength's terms, in the same order).
360
+ def leaf_level(level, k, h)
361
+ log_gamma_k = Math.lgamma(k)[0]
362
+ odds = h.zero? ? nil : Math.log(h) - Math.log(1.0 - h)
363
+ converged = []
364
+ terms = []
365
+ rates = level.map do |pooled, counted, leaves|
366
+ rate, settled = odds.nil? || !counted || leaves.size == 1 ? [ pooled, true ] : null_rate(pooled, leaves, k, odds)
367
+ converged << settled
368
+ leaves.each do |s, t, log_factorial, log_gamma_sk|
369
+ e = t * rate
370
+ terms << [ log_slab_from(log_gamma_sk, s, e, k, log_gamma_k, log_factorial), log_null(s, e, log_factorial) ]
371
+ end
372
+ rate
373
+ end
374
+ { rates: rates, converged: converged, log_likelihood: log_likelihood(terms, h) }
375
+ end
376
+
377
+ # One parent's null rate (fit_leaves) from its pooled rate, at log odds log(h / (1 - h)) with 0 < h < 1 and a
378
+ # count above 0: [rate, converged]. A leaf's weight is its posterior probability of being null, 1 / (1 +
379
+ # exp(d)), d the log odds of slab to null, log(h / (1 - h)) + lgamma(s + k) - lgamma(k) + k log k + E - (k + s)
380
+ # log(k + E), the two parts' lgamma(s + 1) and s log E cancelling. (At h = 0 every weight is 1, with no count
381
+ # every rate is 0, and a lone leaf's weight cancels, so in each case the pooled rate is its own next step.)
382
+ def null_rate(rate, leaves, k, odds)
383
+ LEAF_STEPS.times do
384
+ count = 0.0
385
+ total = 0.0
386
+ leaves.each do |s, t, _log_factorial, _log_gamma_sk, base|
387
+ e = t * rate
388
+ w = 1.0 / (1.0 + Math.exp(odds + base + e - ((k + s) * Math.log(k + e))))
389
+ count += w * s
390
+ total += w * t
391
+ end
392
+ following = count / total
393
+ return [ rate, false ] unless following.finite? && following.positive?
394
+
395
+ settled = (following - rate).abs <= LEAF_CHANGE * rate
396
+ rate = following
397
+ return [ rate, true ] if settled
398
+ end
399
+ [ rate, false ]
400
+ end
401
+
402
+ # sum s / sum t over a parent's leaves, summed in order: every weight 1.
403
+ def pooled_rate(leaves)
404
+ count = 0.0
405
+ total = 0.0
406
+ leaves.each do |s, t|
407
+ count += s
408
+ total += t
409
+ end
410
+ count / total
411
+ end
412
+
413
+ def observable(s, e)
414
+ return unless e.zero? && s.positive?
415
+
416
+ raise ArgumentError, "a count above 0 needs an expected count above 0, got count #{s} and expected 0"
417
+ end
418
+
419
+ # A finite real number, as the other modules read one (a Complex is not real, whatever its imaginary part), and
420
+ # finite as a Float too (an Integer or Rational beyond a Float's range is not).
421
+ def real(value, name)
422
+ number = value.to_f if value.is_a?(Numeric) && value.real? && value.finite?
423
+ return number if number&.finite?
424
+
425
+ raise ArgumentError, "#{name} must be a finite number, got #{value.inspect}"
426
+ end
427
+
428
+ def nonnegative(value, name)
429
+ value = real(value, name)
430
+ raise ArgumentError, "#{name} must be >= 0, got #{value}" if value.negative?
431
+
432
+ value
433
+ end
434
+
435
+ def positive(value, name)
436
+ value = real(value, name)
437
+ raise ArgumentError, "#{name} must be > 0, got #{value}" unless value.positive?
438
+
439
+ value
440
+ end
441
+
442
+ def share(value, name)
443
+ value = real(value, name)
444
+ raise ArgumentError, "#{name} must lie in [0, 1], got #{value}" unless value.between?(0.0, 1.0)
445
+
446
+ value
447
+ end
448
+ end
449
+ end
450
+ end
@@ -0,0 +1,214 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SkiftetStatistical
4
+ # The library's one reading of what a caller hands in besides a number (SkiftetStatistical::Numbers reads those):
5
+ # a random number generator and the values it answers, and a name, each written once, each refusing with an
6
+ # ArgumentError, the one call into a caller's object (`asked`), the one reading of a caller's String (`string`) and
7
+ # the one way a value and a class are shown in a refusal (`shown`, `class_name`). Private to the library: the code
8
+ # new in 0.4.0 asks these; every 0.3.0 module keeps its own readings, as 0.3.0 wrote them.
9
+ #
10
+ # The caller-object contract. Every method the new code calls on a caller's object goes through `asked`: a key
11
+ # part's to_s, a name's hash and eql? (a lookup by it), a list's each, a number's real?, finite? and to_f
12
+ # (Numbers.read), an rng's rand, a status's to_s, an Amount's numbers, Kernel#respond_to? (which asks
13
+ # respond_to_missing?), and, to word a refusal, a value's inspect and an error's message. A StandardError such a
14
+ # call raises, or a NotImplementedError (the error an unfinished method raises), is an ArgumentError naming the
15
+ # input, as an answer of the wrong kind is (or, for the readings that word a refusal, the answer they fall back on).
16
+ # The core values the new code reads from a caller are read by core methods bound to them, never by their own: a
17
+ # String by `string` (String's own b and encoding), a number by Numbers.read, a class's name by Module's own name
18
+ # (`class_name`), a Hash or an Array by Hash's and Array's own methods (`pairs`, `elements`, `fetched`,
19
+ # `with_values`, `merged`), so a subclass's methods, a singleton method or an overridden == are never called, nor a
20
+ # default the Hash holds read. Whatever a caller's object answers is then judged by these one readers. The
21
+ # callables a caller hands in to do its own work (a pass's block, a hold's satisfied_for?) are its own code: what
22
+ # they raise reaches it as raised.
23
+ #
24
+ # An rng is any object that answers rand (asked by Kernel#respond_to?, so an object that answers it only through
25
+ # method_missing, or a BasicObject, is not one); its rand must answer a Float in [0, 1) and its rand(n) an Integer in
26
+ # [0, n). A name is any object that answers hash, eql? and inspect, its hash an Integer, so a Hash can look it up.
27
+ # Each is checked where it is read, before any draw; an answer of rand, which cannot be checked before it is drawn,
28
+ # at the draw that read it (`checked` wraps an rng so that a reader written for a plain rng, such as 0.3.0's
29
+ # Sampler, has each answer checked as it draws it).
30
+ module Inputs
31
+ # Kernel's own methods, bound to the value, so that an object that answers neither (a BasicObject) is read too.
32
+ RESPONDS = Kernel.instance_method(:respond_to?)
33
+ CLASS_OF = Kernel.instance_method(:class)
34
+ # Module's own name, bound to a class, so that a class whose name or to_s is its own is named by Ruby's record.
35
+ MODULE_NAME = Module.instance_method(:name)
36
+ # String's own b and encoding, bound to a caller's String: the one reading of it (`string`).
37
+ STRING_BYTES = String.instance_method(:b)
38
+ STRING_ENCODING = String.instance_method(:encoding)
39
+ # Hash's and Array's own readers, bound to a caller's Hash or Array, so that a subclass's overrides are never
40
+ # called.
41
+ HASH_PAIRS = Hash.instance_method(:to_a)
42
+ HASH_FETCH = Hash.instance_method(:fetch)
43
+ HASH_VALUES = Hash.instance_method(:transform_values)
44
+ HASH_MERGE = Hash.instance_method(:merge)
45
+ ARRAY_ELEMENTS = Array.instance_method(:to_a)
46
+ # What `asked` is told by default to do when the call raises: refuse.
47
+ REFUSE = Object.new.freeze
48
+ private_constant :RESPONDS, :CLASS_OF, :MODULE_NAME, :STRING_BYTES, :STRING_ENCODING, :HASH_PAIRS, :HASH_FETCH,
49
+ :HASH_VALUES, :HASH_MERGE, :ARRAY_ELEMENTS, :REFUSE
50
+
51
+ module_function
52
+
53
+ # THE call into a caller's object, written once: the block's answer, the block calling a method of `value` (`label`
54
+ # names the call: "a part's to_s", "may_lead: a name's lookup"). A StandardError or a NotImplementedError the call
55
+ # raises is an ArgumentError, "<label> raised <its class's name>: <its message> (got <value>)", that error its
56
+ # cause; or, when `otherwise` is given, `otherwise` is answered instead (the readings that word a refusal, and
57
+ # Numbers.read, which reads such a call as no number).
58
+ def asked(value, label, otherwise: REFUSE)
59
+ yield
60
+ rescue StandardError, NotImplementedError => e
61
+ return otherwise unless REFUSE.equal?(otherwise)
62
+
63
+ raise ArgumentError, "#{label} raised #{class_name(e)}#{described(e)} (got #{shown(value)})"
64
+ end
65
+
66
+ # ": <the error's message>", or "" when its message (asked by `asked`) answers no String or raises; the message read
67
+ # by `string` and worded by `words`.
68
+ def described(error)
69
+ text = string(asked(error, "an error's message", otherwise: nil) { error.message })
70
+ nil.equal?(text) ? "" : ": #{words(text)}"
71
+ end
72
+
73
+ # The value as a refusal shows it, the one reading that never refuses, since it words the refusals: a String by
74
+ # its copy's inspect (`string`, so none of its own methods is called); any other value by its inspect (asked by
75
+ # `asked`), read by `string`; for a value that answers none (a BasicObject), whose inspect raises or answers no
76
+ # String, "an object of <its class's name>" (`class_name`). Worded by `words`.
77
+ def shown(value)
78
+ copy = string(value)
79
+ return words(copy.inspect) unless nil.equal?(copy)
80
+
81
+ text = string(asked(value, "an inspect", otherwise: nil) { value.inspect })
82
+ nil.equal?(text) ? "an object of #{class_name(value)}" : words(text)
83
+ end
84
+
85
+ # The name of the value's class, as a refusal words it: read by Kernel#class and Module#name, both bound (never
86
+ # the class's own name or to_s, so building a refusal never calls a caller's method), "an anonymous class" for a
87
+ # class without one; worded by `words`.
88
+ def class_name(value)
89
+ name = MODULE_NAME.bind_call(CLASS_OF.bind_call(value))
90
+ nil.equal?(name) ? "an anonymous class" : words(string(name))
91
+ end
92
+
93
+ # THE reading of a caller's String, written once (a key part or its to_s answer, an item name, a status or its
94
+ # to_s answer, a name in a list, and the words of a refusal): a plain, frozen String of the same bytes in the same encoding,
95
+ # copied by String's own b and encoding bound to it, so a subclass's methods, a singleton method (a frozen String
96
+ # keeps those) or an overridden == are never called; nil for a value that is not a String. What the new code reads
97
+ # of the String (==, include?, inspect, its encoding, its bytes) it then reads of the copy, whose methods are
98
+ # String's own.
99
+ def string(value)
100
+ return nil unless String === value
101
+
102
+ STRING_BYTES.bind_call(value).force_encoding(STRING_ENCODING.bind_call(value)).freeze
103
+ end
104
+
105
+ # THE reading of a value as text, written once (a key part, a status): a String by `string`, so none of its own
106
+ # methods is called, its to_s included; any other value by its to_s (Kernel#respond_to? and the call asked by
107
+ # `asked`, `label` naming the value, so a to_s that raises is an ArgumentError), its answer read by `string`; nil
108
+ # when the value answers no to_s or its to_s answers no String.
109
+ def text(value, label)
110
+ copy = string(value)
111
+ return copy unless nil.equal?(copy)
112
+ return nil unless responds?(value, :to_s, label)
113
+
114
+ string(asked(value, "#{label}'s to_s") { value.to_s })
115
+ end
116
+
117
+ # A copy (`string`) as a refusal sets it among its words: as it is when its bytes are ASCII in an ASCII-compatible
118
+ # encoding, or valid in the encoding String#inspect answers in (the default internal encoding, else the default
119
+ # external, US-ASCII when that is not ASCII-compatible); else its inspect, its characters escaped in that encoding.
120
+ # So the words of one refusal never meet in incompatible encodings, and building it never raises.
121
+ def words(text)
122
+ return text if text.ascii_only?
123
+
124
+ encoding = Encoding.default_internal || Encoding.default_external
125
+ encoding = Encoding::US_ASCII unless encoding.ascii_compatible?
126
+ text.encoding == encoding && text.valid_encoding? ? text : text.inspect
127
+ end
128
+
129
+ # A caller's Hash's [key, value] pairs, in its order, by Hash's own to_a: a new Array.
130
+ def pairs(hash)
131
+ HASH_PAIRS.bind_call(hash)
132
+ end
133
+
134
+ # A caller's Hash's value at `key`, or `missing` when it holds none, by Hash's own fetch (a default the Hash holds
135
+ # is never read). Looking a caller's key up calls its hash and eql?: the caller asks this through `asked`.
136
+ def fetched(hash, key, missing)
137
+ HASH_FETCH.bind_call(hash, key, missing)
138
+ end
139
+
140
+ # A caller's Hash with each value replaced by the block's answer for it, by Hash's own transform_values: a new
141
+ # Hash of its keys as given, never hashed again.
142
+ def with_values(hash, &)
143
+ HASH_VALUES.bind_call(hash, &)
144
+ end
145
+
146
+ # A caller's Hash merged with `other`, by Hash's own merge.
147
+ def merged(hash, other)
148
+ HASH_MERGE.bind_call(hash, other)
149
+ end
150
+
151
+ # A caller's Array's elements, by Array's own to_a: the Array itself, or for a subclass's, a new plain Array.
152
+ def elements(array)
153
+ ARRAY_ELEMENTS.bind_call(array)
154
+ end
155
+
156
+ # Whether the value answers the method, by Kernel#respond_to? (a BasicObject answers nothing), asked by `asked`
157
+ # (respond_to? asks the value's respond_to_missing?), `label` naming the value.
158
+ def responds?(value, method, label = "a value")
159
+ asked(value, "#{label}'s respond_to?(:#{method})") { RESPONDS.bind_call(value, method) }
160
+ end
161
+
162
+ # The rng, when it answers rand; else an ArgumentError naming `label`.
163
+ def rng(value, label = "rng")
164
+ return value if responds?(value, :rand, label)
165
+
166
+ raise ArgumentError, "#{label} must answer rand (got #{shown(value)})"
167
+ end
168
+
169
+ # The rng's next rand (asked by `asked`), when it is a Float in [0, 1); else an ArgumentError.
170
+ def uniform(rng)
171
+ value = asked(rng, "rng.rand") { rng.rand }
172
+ return value if Float === value && value >= 0.0 && value < 1.0
173
+
174
+ raise ArgumentError, "rng.rand must answer a Float in [0, 1) (got #{shown(value)})"
175
+ end
176
+
177
+ # The rng's next rand(size) (asked by `asked`), when it is an Integer in [0, size); else an ArgumentError.
178
+ def index(rng, size)
179
+ value = asked(rng, "rng.rand(#{size})") { rng.rand(size) }
180
+ return value if Integer === value && value >= 0 && value < size
181
+
182
+ raise ArgumentError, "rng.rand(#{size}) must answer an Integer in [0, #{size}) (got #{shown(value)})"
183
+ end
184
+
185
+ # The rng (by `rng`), wrapped so that each of its answers is checked when it is drawn: rand by `uniform`, rand(n)
186
+ # by `index`. The wrapper answers the rng's own stream, one rand of the rng a rand of its own; an rng already
187
+ # wrapped is answered as it is.
188
+ def checked(value, label = "rng")
189
+ Checked === value ? value : Checked.new(rng(value, label))
190
+ end
191
+
192
+ # Whether the value can name something a Hash looks up: it answers hash, eql? and inspect, and its hash answers
193
+ # an Integer (a Hash raises TypeError on any other hash, and calls eql? on a name whose hash matches a key's). Its
194
+ # hash is asked by `asked`, `label` naming where the name was given: a hash that raises is an ArgumentError.
195
+ def name?(value, label = "a name")
196
+ responds?(value, :hash, label) && responds?(value, :eql?, label) && responds?(value, :inspect, label) &&
197
+ Integer === asked(value, "#{label}: a name's hash") { value.hash }
198
+ end
199
+
200
+ # An rng whose answers are checked as they are drawn (`checked`).
201
+ class Checked
202
+ def initialize(rng)
203
+ @rng = rng
204
+ end
205
+
206
+ # The rng's rand, checked by `uniform`; with a size, its rand(size), checked by `index`.
207
+ def rand(size = nil)
208
+ nil.equal?(size) ? Inputs.uniform(@rng) : Inputs.index(@rng, size)
209
+ end
210
+ end
211
+ private_constant :Checked
212
+ end
213
+ private_constant :Inputs
214
+ end