skiftet_statistical 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +308 -8
  3. data/README.md +1416 -39
  4. data/lib/skiftet_statistical/additive_contrasts.rb +894 -0
  5. data/lib/skiftet_statistical/bandit/allocator.rb +477 -63
  6. data/lib/skiftet_statistical/bandit/checks.rb +128 -0
  7. data/lib/skiftet_statistical/bandit/decay.rb +86 -0
  8. data/lib/skiftet_statistical/bandit/draw.rb +179 -0
  9. data/lib/skiftet_statistical/bandit/normal_argmax.rb +209 -0
  10. data/lib/skiftet_statistical/bandit/pass.rb +94 -0
  11. data/lib/skiftet_statistical/bandit/posteriors.rb +497 -0
  12. data/lib/skiftet_statistical/bandit/prior.rb +264 -0
  13. data/lib/skiftet_statistical/bandit/probabilities.rb +929 -0
  14. data/lib/skiftet_statistical/bandit/ranking.rb +552 -0
  15. data/lib/skiftet_statistical/bandit/summary.rb +152 -0
  16. data/lib/skiftet_statistical/bandit/tallies.rb +297 -0
  17. data/lib/skiftet_statistical/bandit.rb +3 -3
  18. data/lib/skiftet_statistical/beta_argmax.rb +526 -0
  19. data/lib/skiftet_statistical/beta_function.rb +28 -1
  20. data/lib/skiftet_statistical/cholesky.rb +244 -0
  21. data/lib/skiftet_statistical/cusum.rb +183 -0
  22. data/lib/skiftet_statistical/descriptive.rb +3 -3
  23. data/lib/skiftet_statistical/gauss_legendre.rb +73 -0
  24. data/lib/skiftet_statistical/hierarchical_rates.rb +450 -0
  25. data/lib/skiftet_statistical/inputs.rb +214 -0
  26. data/lib/skiftet_statistical/linear_value.rb +364 -0
  27. data/lib/skiftet_statistical/numbers.rb +98 -0
  28. data/lib/skiftet_statistical/prequential.rb +206 -0
  29. data/lib/skiftet_statistical/shared_credit.rb +811 -0
  30. data/lib/skiftet_statistical/significance.rb +5 -6
  31. data/lib/skiftet_statistical/version.rb +1 -1
  32. data/lib/skiftet_statistical.rb +43 -4
  33. metadata +37 -11
@@ -0,0 +1,929 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../numbers"
4
+ require_relative "../inputs"
5
+ require_relative "../beta_argmax"
6
+ require_relative "../significance"
7
+ require_relative "checks"
8
+ require_relative "draw"
9
+ require_relative "normal_argmax"
10
+ require_relative "posteriors"
11
+ require_relative "ranking"
12
+
13
+ module SkiftetStatistical
14
+ class Bandit
15
+ # The posterior set's odds: the probability of each head a pick answers, of each order and of each item at each
16
+ # position that `rank` answers, all under the rule that made the draw, and the probability that a two-layer draw
17
+ # shows an item.
18
+ #
19
+ # set = SkiftetStatistical::Bandit::Posteriors.beta({ "a" => [ 1.0, 1.0 ], "b" => [ 2.0, 1.0 ] })
20
+ # set.pick_probabilities[:heads] # => { "a" => 1/3, "b" => 2/3 }
21
+ # set.pick_probabilities(explore: 0.1)[:heads] # => { "a" => 0.05 + 0.9 * 1/3, "b" => 0.05 + 0.9 * 2/3 }
22
+ # set.order_probability(%w[b a]) # => { probability: 2/3, method: :exact, standard_error: 0.0 }
23
+ # set.positions[:probabilities] # => { "a" => [1/3, 2/3], "b" => [2/3, 1/3] }
24
+ # set.order_summary(%w[b a]) # => { probability: 2/3, positions: { "a" => [...], ... } }
25
+ # set.rank(key: "k1").probability # => the probability of the order that key drew
26
+ # SkiftetStatistical::Bandit::Posteriors.mixture(
27
+ # shares: { "x" => 0.25, "y" => 0.75 },
28
+ # probabilities: { "x" => { "a" => 0.5, "b" => 0.5 }, "y" => { "a" => 0.2, "c" => 0.8 } }
29
+ # ) # => { "a" => 0.275, "b" => 0.125, "c" => 0.6 }
30
+ #
31
+ # THE PICK (`pick_probabilities`), under the rule `pick` draws by: `shuffle` and `explore` (each in [0, 1], summing
32
+ # to at most 1), `may_lead` (nil: every item) and `min_views` (0). The contenders C are the items `may_lead` names
33
+ # (every item when it is nil or names none), the eligible E the contenders with at least `min_views` views. The
34
+ # head is item c with probability
35
+ #
36
+ # shuffle / |C| [c in C] + explore / |E| [c in E, E not empty] + (1 - shuffle - explore [E not empty]) P(c)
37
+ #
38
+ # with P(c) the probability that c's draw is the largest among C's: BetaArgmax.probabilities or
39
+ # NormalArgmax.probabilities on the scaled pairs (`parameters`), through the set's one P(largest) by kind, up to
40
+ # EXACT_PICK_MAX contenders, each by its quadrature within its stated bound, not to the last digit: BetaArgmax's
41
+ # within TAIL (1e-10) a member, and NormalArgmax's, each member in its own standardised variable, within
42
+ # 2 k (k + 1) TAIL + 2e-12 of the exact probability for k contenders (1.2e-9 for two, 8.4e-8 for 20). Above
43
+ # EXACT_PICK_MAX contenders P(c) is the stated Monte Carlo below, and the rest of the formula stands as it is. When
44
+ # no contender has `min_views` views the exploration share goes back to the draw, as the pick does. Every source of
45
+ # the pick (a key, given uniforms, an RNG) draws exactly from the posteriors on the scaled pairs, and compares two
46
+ # draws as the posteriors order them (a Beta by its position or its value, a Normal by the exact sign of its mean
47
+ # + sd * z against the other's, ranking.rb), so these are, within that bound, the odds of a pick from any of them.
48
+ # The answer:
49
+ # - `heads`: { item => P(head = item) } for every item, in item order (0.0 for an item that may not lead);
50
+ # - `leaders`: the pick's leader is its head when the head has at least `min_views` views, else nil, so each item
51
+ # with at least `min_views` views gets its head's probability and nil gets the heads of the items below it
52
+ # (0.0 when there are none); an empty set answers no heads and a leader nil with probability 1;
53
+ # - `method`: :exact (the quadrature, not a Monte Carlo), and `standard_error`: 0.0; or, above EXACT_PICK_MAX
54
+ # contenders with a share left to the draw, :monte_carlo and the largest standard error of a head or a leader.
55
+ #
56
+ # THE ORDERS (`order_probability(order)`) and THE POSITIONS (`positions`), under the rule `rank` draws by: the
57
+ # pick's four keywords and `pinned` ({ position => item }) and `top`. Pinned items keep their positions; the n free
58
+ # items fill the other positions in order, each free rank on the next free slot. `order` is a list of distinct
59
+ # items, as `rank` answers them (pinned items at their positions): its probability is that of the rule's whole
60
+ # order beginning with it, so for an order of `top` items (every item when top is nil) it is the probability that
61
+ # `rank` answers it, and `top` itself changes no odds. An order whose pinned slot holds another item, or that puts
62
+ # a pinned item on a free slot, has probability 0; the empty order 1. With s1, ..., sm the free items the order
63
+ # names, in its order, its probability is
64
+ #
65
+ # shuffle / (n (n - 1) ... (n - m + 1))
66
+ # + explore [E not empty] [s1 in E] / |E| * T(s2, ..., sm among the free items less s1)
67
+ # + (1 - shuffle - explore [E not empty]) * T(s1, ..., sm among the free items),
68
+ #
69
+ # T(s1, ..., sm among S) = P(X_s1 > ... > X_sm > every other item of S), and `positions` answers P(item at slot)
70
+ # on the same parts: the shuffled part 1/n at every free rank; the explored part 1/|E| at rank 0 for an eligible
71
+ # item, and at rank l >= 1 the sum over h in E, h != i, of 1/|E| * P(i at rank l - 1 by draw among the free items
72
+ # less h); the drawn part P(i at rank l by draw among the free items). A pinned item is 1 at its own slot.
73
+ #
74
+ # Each is exact, by a quadrature, when no free item is barred from leading (`may_lead` names every free item, or
75
+ # none) and there are at most EXACT_ORDER_MAX free items; otherwise it is the stated Monte Carlo below. A rank by
76
+ # draw at the head is P(largest), the set's one P(largest) by kind (above); every other is a Stieltjes sum against
77
+ # exact distribution increments, each level of a chain and each item's position sums integrated over its own
78
+ # item's points, in a coordinate where that item's own points keep their digits (on the scaled pairs). Each item's
79
+ # points are -infinity, +infinity and the free items' quantiles at the ORDER_LEVELS - 1 evenly spaced levels
80
+ # inside (0, 1) (for Normals also at 0 and 1, below), sorted, each once.
81
+ # - Betas share one grid, in the position of BetaArgmax.position (the key ranking.rb compares their draws by:
82
+ # log(2 x) up to 1/2, then -log(2 (1 - x))), where a Beta's points keep their digits. Every item's distribution
83
+ # function is read there by BetaArgmax.distribution (the very function its P(largest) and its draws read), each
84
+ # half from its own end as BetaArgmax reads its halves, and below 2**-1023 from that end, where a Float holds
85
+ # fewer digits of x and then none (and where a parameter near SMALLEST_PARAMETER holds nearly all of its mass),
86
+ # by its leading term, a power of x scaled from BetaArgmax.distribution at 2**-1023.
87
+ # - Normals have no such shared coordinate within the input domain: x itself rounds a narrow item's points
88
+ # together at a large mean, x less the largest mean rounds two narrow items near 0 together beside a far item,
89
+ # and a scale by a standard deviation sends a wide item's points past a Float's range. So each item has its own
90
+ # points in its own standardised variable z = (x - mean) / sd, as NormalArgmax.probabilities integrates each
91
+ # member: every free item's quantiles, at the levels inside (0, 1) and at 0 and 1 read as a quantile draw's
92
+ # uniform is (Draw.open_uniform, 2**-53 within the ends), carried into z as ((the other's mean - the item's mean)
93
+ # + the other's sd * the standard quantile) / the item's sd (the means' difference formed first; a point beyond a
94
+ # Float's range is left out), and beside each item's carried points the second Float below the lowest and above
95
+ # the highest, so that an item whose points round together in z rises wholly between two points. The ends keep
96
+ # an item's own mass below its first quantile on a finite point, where the other items' F hold their values, not
97
+ # the 0 every F reads at -infinity. Every item's distribution function is read at the item's points as
98
+ # Phi(((the item's mean - the other's mean) + the item's sd * z) / the other's sd).
99
+ # So the chain, each level on its own item's points,
100
+ #
101
+ # H(x) = the product of the other items' F (1 when none); then, for k = m down to 1,
102
+ # H_k(x_j) = H_k(x_(j-1)) + (F_sk(x_j) - F_sk(x_(j-1))) * (H_(k+1)(x_(j-1)) + H_(k+1)(x_j)) / 2
103
+ #
104
+ # gives T = H_1 at the top (an order's last item, when no other is left, is the rest, and a chain of one item is
105
+ # its P(largest)). H_(k+1), held on s_(k+1)'s points, is read at s_k's: on the Betas' one grid as it stands; for
106
+ # Normals at each of s_k's points mapped into s_(k+1)'s z (the means' difference formed first, then plus s_k's sd
107
+ # times z, over s_(k+1)'s sd), linearly in F_s(k+1) between the two of s_(k+1)'s points around it, which keeps
108
+ # it monotone and exact where the point is one of s_(k+1)'s (H rises between two points by the increment of F
109
+ # times a mean of what it integrates, so where F does not move H does not either). P(i at rank l by draw) sums each
110
+ # increment of F_i on i's points times the mean, at the interval's two ends, of the z**l coefficient of the
111
+ # product over j != i of (F_j + z (1 - F_j)) (the explored part the same over the items less h, summed over h).
112
+ # Each sum is taken on the points and on their coarse part (the infinities, the quantiles at every other level and,
113
+ # for Normals, the points beside each item's carried points) and extrapolated, fine + (fine - coarse) / 3
114
+ # (Richardson), which takes out the trapezoid's error that falls as the square of the spacing. Each item's mass is
115
+ # split evenly between its own quantiles, so no item is sampled coarsely, a parameter below 1 included, where a
116
+ # trapezoid of the density reads up to 7e-3 off: at 400 levels an item the orders and positions read within 4e-6
117
+ # of the closed forms of Beta(a, 1) and Beta(1, b) items, which race as Plackett-Luce (800 random sets of three and
118
+ # four, a and b from 1e-6 to 10), three identical items every order 1/n! and every position 1/n within 1e-10 at
119
+ # any scale, two narrow Normals beside a far one (deviations 1e-12 to 1e-7, near 0 or 1e9, the far one 1e6 ahead
120
+ # with a deviation of 1 or 1e10) within 1.2e-6 of their closed forms, random Normal sets of three to five (means
121
+ # up to 5e7 either side of 0, deviations 1e-6 to 1e6) with their positions' rows and columns and their orders by
122
+ # head within 3e-6 of 1 and of the pick's odds, and all of them within 1e-4 of the exact figures (the library's
123
+ # stated tolerance), answered as Floats in [0, 1] (`odds_bounded`, the odds' one clamp).
124
+ #
125
+ # THE MONTE CARLO, stated, wherever an odds call has no exact rule (`draws` and `seed` read first, by the bandit's
126
+ # one reading of a whole number, Checks.integer: draws at least 1, seed any Integer within a Float's range):
127
+ # `draws` draws from the RNG source on Random.new(seed), so the same call answers the same figures, each on the
128
+ # scaled pairs (`parameters`), the one pair the exact odds and every draw (`rank`, `pick` and `values`, from any
129
+ # source) read. For the pick, P(c is C's largest draw) is the share of those draws in which c's draw (one Sampler
130
+ # draw per contender, in item order, the set's one RNG draw) is the largest by the order rule's own choice of a
131
+ # first largest, compared by the key the rule compares (a Normal's exact mean + sd * z); the standard error of a
132
+ # head is its drawn share times sqrt(q (1 - q) / draws). For orders and positions, the rule itself visits `draws`
133
+ # times (the order rule's own ordering, as `rank(rng:)` orders), and each figure is the share of visits that answer
134
+ # it, its standard error sqrt(p (1 - p) / draws). The answer's
135
+ # `method` is :monte_carlo and its `standard_error` the largest of its figures'. A Beta parameter far below 1 draws
136
+ # 0.0 in a share of Sampler's draws (its Gamma draw underflows: about a tenth of them at 0.003), where two such
137
+ # draws tie and the tie goes to the earlier item, so below about 0.003 the Monte Carlo reads that tie rule as well
138
+ # as the posterior. It refuses what the exact odds refuse (below), at every size.
139
+ #
140
+ # THE POSTERIOR ORDER (`order_summary(order)`): the posterior alone, no rule: P(X_s1 > ... > X_sm > every other
141
+ # item), that the order is the best one (T above, over every item), and every item's P(at each position) by draw,
142
+ # exact as above, for a set of at most EXACT_ORDER_MAX items.
143
+ #
144
+ # Choice#probability and Order#probability answer the odds of the head or the order they hold under the rule they
145
+ # were made by (the set, the rule and the cache each carries): `pick_probabilities` of the head (1.0 for no head,
146
+ # an empty set) and `order_probability` of the items, at the default draws and seed where those are a Monte Carlo;
147
+ # computed on the first call and kept.
148
+ #
149
+ # Refused, as ArgumentErrors before anything is computed: `draws` that is not an Integer of at least 1, or `seed`
150
+ # that is not an Integer, within a Float's range; a rule `rank` or `pick` refuses (read by the order rule's one
151
+ # reader); an `order` that is not a list of items or that names an item twice; `order_summary` of more than
152
+ # EXACT_ORDER_MAX items; and, by the set's one refusal, a Beta parameter below BetaArgmax::SMALLEST_PARAMETER on the
153
+ # scaled pairs of any free item (one that may not lead included; for the pick every item), the scope a keyed draw
154
+ # of the same set and rule refuses, at every size: the exact odds and the stated Monte Carlo alike (the pick at any
155
+ # number of contenders, the orders and positions with or without items barred from leading, the posterior order);
156
+ # a parameter at it is taken.
157
+ #
158
+ # THE MIXTURE (`Posteriors.mixture`): a first layer draws branch b with probability shares[b], then a second layer
159
+ # shows item c with probability probabilities[b][c]. Item c is shown with probability
160
+ #
161
+ # sum over b of shares[b] * probabilities[b][c]
162
+ #
163
+ # (an item a branch does not name counts 0 there), answered as { item => Float } in the order the items are first
164
+ # named, branch by branch. Branches and items may be any keys a Hash can look up (the library's one test of a name,
165
+ # Inputs.name?), read as given (Symbols and Strings are different keys). Every branch of `shares` must have its
166
+ # probabilities, a Hash (empty for a branch that shows nothing), and `probabilities` may name no branch that
167
+ # `shares` does not; every share and probability is a number >= 0 (by the library's one number reader, in the input
168
+ # domain, Numbers::LARGEST), so no product or sum leaves a Float. An ArgumentError otherwise. Every answer passes
169
+ # the bandit's one check of an answer (Checks.finite_answer).
170
+ class Posteriors
171
+ # The quantile levels an item, evenly spaced, at which the free items split the order odds' grid.
172
+ ORDER_LEVELS = 400
173
+ # Half a Float's smallest normal, 2**-1023: at and below it a Beta's distribution function is its leading term.
174
+ ORDER_FLOOR = 2.0**-1023
175
+ ORDER_LOG_FLOOR = Math.log(ORDER_FLOOR)
176
+ # What the odds compute, exact or by their stated Monte Carlo, and the posterior order, in the words of GB6's
177
+ # refusal: one refusal at every size, over the items a keyed draw of the same set and rule refuses.
178
+ ODDS_PICK_EXACT = "the exact pick odds on the scaled pairs"
179
+ ODDS_PICK_COUNTED = "the pick odds' stated Monte Carlo, which refuses what the exact pick odds on the scaled " \
180
+ "pairs refuse"
181
+ ORDER_EXACT = "the exact order and position odds on the scaled pairs"
182
+ ORDER_COUNTED = "the order and position odds' stated Monte Carlo, which refuses what the exact order and " \
183
+ "position odds on the scaled pairs refuse"
184
+ ORDER_POSTERIOR = "the posterior order and positions on the scaled pairs"
185
+ private_constant :ORDER_LEVELS, :ORDER_FLOOR, :ORDER_LOG_FLOOR, :ODDS_PICK_EXACT, :ODDS_PICK_COUNTED,
186
+ :ORDER_EXACT, :ORDER_COUNTED, :ORDER_POSTERIOR
187
+
188
+ class << self
189
+ # The probability that a two-layer draw shows each item: { item => sum over branches b of shares[b] *
190
+ # probabilities[b][item] }.
191
+ def mixture(shares:, probabilities:)
192
+ odds_refuse_unpaired(shares, probabilities)
193
+ shown = {}
194
+ Inputs.pairs(shares).each do |branch, given_share|
195
+ share = Checks.nonnegative(given_share, "shares[#{Inputs.shown(branch)}]")
196
+ label = "probabilities[#{Inputs.shown(branch)}]"
197
+ Inputs.pairs(Checks.lookup(probabilities, branch, "probabilities")).each do |item, given|
198
+ odds_key(item, label)
199
+ probability = Checks.nonnegative(given, "#{label}[#{Inputs.shown(item)}]")
200
+ sum = Checks.lookup(shown, item, label, 0.0) + (share * probability)
201
+ Inputs.asked(item, "#{label}: an item's hash") { shown[item] = sum }
202
+ end
203
+ end
204
+ Checks.finite_answer(shown.values, "the shares and probabilities", shown)
205
+ end
206
+
207
+ private
208
+
209
+ # Both Hashes, every branch of `shares` with a Hash of probabilities, and no branch in `probabilities` that
210
+ # `shares` does not name, each branch looked up by the bandit's one lookup in a caller's Hash (Checks.lookup: a
211
+ # default the Hash holds is never read, and the branch's hash and eql? are asked by Inputs.asked).
212
+ def odds_refuse_unpaired(shares, probabilities)
213
+ raise ArgumentError, "shares must be a Hash of branch => share (got #{Inputs.class_name(shares)})" unless Hash === shares
214
+ unless Hash === probabilities
215
+ raise ArgumentError, "probabilities must be a Hash of branch => { item => probability } " \
216
+ "(got #{Inputs.class_name(probabilities)})"
217
+ end
218
+
219
+ branches = Inputs.pairs(shares).map(&:first)
220
+ named = Inputs.pairs(probabilities).map(&:first)
221
+ branches.each { |branch| odds_key(branch, "shares") }
222
+ named.each { |branch| odds_key(branch, "probabilities") }
223
+ branches.each do |branch|
224
+ found = Checks.lookup(probabilities, branch, "probabilities")
225
+ next if Hash === found
226
+
227
+ raise ArgumentError, "probabilities must name a Hash of item => probability for the branch " \
228
+ "#{Inputs.shown(branch)} (got #{Inputs.shown(found)})"
229
+ end
230
+ stranger = named.find { |branch| !Checks.holds?(shares, branch, "shares") }
231
+ raise ArgumentError, "probabilities names the branch #{Inputs.shown(stranger)}, which has no share" unless stranger.nil?
232
+ end
233
+
234
+ # A branch or an item as given, when a Hash can look it up (the library's one test of a name, Inputs.name?:
235
+ # it answers hash, with an Integer, eql? and inspect, a hash that raises refused there); else an ArgumentError
236
+ # naming `label`.
237
+ def odds_key(key, label)
238
+ return key if Inputs.name?(key, label)
239
+
240
+ raise ArgumentError, "#{label}: #{Inputs.shown(key)} cannot be looked up (a key must answer hash, with an " \
241
+ "Integer, eql? and inspect)"
242
+ end
243
+ end
244
+
245
+ # The odds of the pick under its rule (above): { heads:, leaders:, method:, standard_error: }, exact up to
246
+ # EXACT_PICK_MAX contenders, the stated Monte Carlo of P(largest) above.
247
+ def pick_probabilities(draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0)
248
+ Checks.integer(draws, "draws", minimum: 1)
249
+ Checks.integer(seed, "seed")
250
+ rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: {},
251
+ top: nil)
252
+ candidates = contenders(rule[:may_lead])
253
+ exact = candidates.size <= EXACT_PICK_MAX
254
+ refuse_below_smallest_parameter(computing: exact ? ODDS_PICK_EXACT : ODDS_PICK_COUNTED)
255
+ return { heads: {}, leaders: { nil => 1.0 }, method: :exact, standard_error: 0.0 } if candidates.empty?
256
+
257
+ shares = odds_shares(candidates, rule)
258
+ drawn = shares.last
259
+ counted = !exact && drawn.positive?
260
+ largest = if counted
261
+ order_largest_counted(candidates, draws, seed)
262
+ elsif drawn.positive?
263
+ largest_draw(candidates).first
264
+ else
265
+ Array.new(candidates.size, 0.0)
266
+ end
267
+ heads = odds_heads(candidates, rule, shares, largest)
268
+ leaders = odds_leaders(heads, rule[:min_views])
269
+ error = counted ? odds_error(candidates, rule, drawn, largest, draws) : 0.0
270
+ Checks.finite_answer(heads.values + leaders.values << error, "the parameters and the rule",
271
+ { heads: heads, leaders: leaders, method: counted ? :monte_carlo : :exact,
272
+ standard_error: error })
273
+ end
274
+
275
+ # The probability that the rule's whole order begins with `order` (above): { probability:, method:,
276
+ # standard_error: }, exact without items barred from leading up to EXACT_ORDER_MAX free items, else the stated
277
+ # Monte Carlo.
278
+ def order_probability(order, draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0,
279
+ pinned: {}, top: nil)
280
+ Checks.integer(draws, "draws", minimum: 1)
281
+ Checks.integer(seed, "seed")
282
+ rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: pinned,
283
+ top: top)
284
+ named = order_named(order, "order")
285
+ free = rank_free(rule)
286
+ exact = order_exact?(rule, free)
287
+ refuse_below_smallest_parameter(free, computing: exact ? ORDER_EXACT : ORDER_COUNTED)
288
+ if exact
289
+ sequence = order_sequence(named, rule[:pinned])
290
+ probability = nil.equal?(sequence) ? 0.0 : order_exact(sequence, free, rule)
291
+ answer = { probability: probability, method: :exact, standard_error: 0.0 }
292
+ else
293
+ probability, error = order_counted(named, rule, draws, seed)
294
+ answer = { probability: probability, method: :monte_carlo, standard_error: error }
295
+ end
296
+ Checks.finite_answer(answer.values_at(:probability, :standard_error), "the parameters and the rule", answer)
297
+ end
298
+
299
+ # P(item at each slot) under the rule (above): { probabilities: { item => [Float, one a slot] }, method:,
300
+ # standard_error: }, every item in item order, exact as the orders are, else the stated Monte Carlo. `top` only
301
+ # cuts what `rank` answers, so the table is the whole order's.
302
+ def positions(draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0, pinned: {},
303
+ top: nil)
304
+ Checks.integer(draws, "draws", minimum: 1)
305
+ Checks.integer(seed, "seed")
306
+ rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: pinned,
307
+ top: top)
308
+ free = rank_free(rule)
309
+ exact = order_exact?(rule, free)
310
+ refuse_below_smallest_parameter(free, computing: exact ? ORDER_EXACT : ORDER_COUNTED)
311
+ if exact
312
+ table = order_slotted(order_ranks(free, rule), free, rule[:pinned])
313
+ error = 0.0
314
+ method = :exact
315
+ else
316
+ table, error = order_counted_positions(rule, draws, seed)
317
+ method = :monte_carlo
318
+ end
319
+ Checks.finite_answer(table.values.flatten << error, "the parameters and the rule",
320
+ { probabilities: table, method: method, standard_error: error })
321
+ end
322
+
323
+ # The posterior alone (above): { probability: P(X_s1 > ... > X_sm > every other item), positions: { item =>
324
+ # [P(at each position by draw)] } }, exact, for a set of at most EXACT_ORDER_MAX items.
325
+ def order_summary(order)
326
+ named = order_named(order, "order")
327
+ if @items.size > EXACT_ORDER_MAX
328
+ raise ArgumentError, "order_summary computes the posterior order exactly among at most #{EXACT_ORDER_MAX} " \
329
+ "items (the set holds #{@items.size})"
330
+ end
331
+
332
+ refuse_below_smallest_parameter(computing: ORDER_POSTERIOR)
333
+ rule = rank_rule(shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0, pinned: {}, top: nil)
334
+ probability = order_exact(named, @items, rule)
335
+ table = order_slotted(order_ranks(@items, rule), @items, rule[:pinned])
336
+ Checks.finite_answer(table.values.flatten << probability, "the parameters",
337
+ { probability: probability, positions: table })
338
+ end
339
+
340
+ private
341
+
342
+ # [eligible ({ item => true }, in item order), the explored share, the drawn share] for the contenders under the
343
+ # rule: the explored share is explore when an item is eligible, else 0 (it goes back to the draw); the drawn
344
+ # share what is left, never below 0. The pick's heads and the orders' parts read their shares here.
345
+ def odds_shares(candidates, rule)
346
+ eligible = candidates.select { |item| eligible?(item, rule[:min_views]) }.to_h { |item| [ item, true ] }
347
+ explored = eligible.empty? ? 0.0 : rule[:explore]
348
+ [ eligible, explored, [ 1.0 - rule[:shuffle] - explored, 0.0 ].max ]
349
+ end
350
+
351
+ # { item => P(head = item) } for every item: the shuffled share spread over the contenders, the explored share
352
+ # over the eligible, the drawn share by `largest` (P(largest draw) of each contender, in their order), held to
353
+ # [0, 1] (`odds_bounded`).
354
+ def odds_heads(candidates, rule, shares, largest)
355
+ eligible, explored, drawn = shares
356
+ heads = @items.to_h { |item| [ item, 0.0 ] }
357
+ candidates.each_with_index do |item, index|
358
+ head = (rule[:shuffle] / candidates.size) + (eligible.key?(item) ? explored / eligible.size : 0.0) +
359
+ (drawn * largest[index])
360
+ heads[item] = odds_bounded(head)
361
+ end
362
+ heads
363
+ end
364
+
365
+ # THE clamp of the odds, written once: a probability held to [0, 1], so a sum that rounds below 0 reads 0 and
366
+ # one that rounds above 1 reads 1 (the pick's heads, the orders' chains and parts, the positions' cells).
367
+ def odds_bounded(probability)
368
+ probability.clamp(0.0, 1.0)
369
+ end
370
+
371
+ # { item => P(leader = item) } for each item with at least min_views views, then nil => the heads of the rest.
372
+ def odds_leaders(heads, min_views)
373
+ leaders = {}
374
+ without = 0.0
375
+ heads.each do |item, probability|
376
+ if eligible?(item, min_views)
377
+ leaders[item] = probability
378
+ else
379
+ without += probability
380
+ end
381
+ end
382
+ leaders[nil] = without
383
+ leaders
384
+ end
385
+
386
+ # The largest standard error of a head or a leader whose P(largest) part is the Monte Carlo's: the drawn share
387
+ # times sqrt(q (1 - q) / draws), q a contender's share of the draws, or for the leader nil the share summed over
388
+ # the contenders below min_views.
389
+ def odds_error(candidates, rule, drawn, largest, draws)
390
+ without = 0.0
391
+ errors = candidates.each_with_index.map do |item, index|
392
+ without += largest[index] unless eligible?(item, rule[:min_views])
393
+ order_binomial_error(largest[index], draws)
394
+ end
395
+ drawn * (errors << order_binomial_error(without, draws)).max
396
+ end
397
+
398
+ # The Monte Carlo of P(largest) among the contenders: over `draws` draws from Random.new(seed), one Sampler draw
399
+ # per contender in item order (the set's one RNG draw, `sampler_draw`, on the scaled pairs the exact pick odds
400
+ # read), the share in which each contender's draw is the largest by the order rule's own choice of a first
401
+ # largest (`rank_first`, ties to the earlier item) on the keys the rule compares.
402
+ def order_largest_counted(candidates, draws, seed)
403
+ sampler = checked_sampler(Random.new(seed))
404
+ keys = candidates.to_h { |item| [ item, 0.0 ] }
405
+ counts = candidates.to_h { |item| [ item, 0 ] }
406
+ draws.times do
407
+ candidates.each { |item| keys[item] = sampler_draw(sampler, item).first }
408
+ counts[rank_first(candidates, keys, :>)] += 1
409
+ end
410
+ candidates.map { |item| counts.fetch(item).fdiv(draws) }
411
+ end
412
+
413
+ # sqrt(p (1 - p) / draws), a share's standard error over `draws` draws.
414
+ def order_binomial_error(share, draws)
415
+ Math.sqrt([ share * (1.0 - share), 0.0 ].max / draws)
416
+ end
417
+
418
+ # The items a caller's order names, read by the set's one reading of a list of item names (`listed_items`), as
419
+ # the set's own Strings in the order's order, frozen; an item named twice is an ArgumentError.
420
+ def order_named(order, label)
421
+ _chosen, named = listed_items(order, label, @items, [])
422
+ seen = {}
423
+ named.each do |item|
424
+ raise ArgumentError, "#{label}: #{Inputs.shown(item)} is named twice" if seen.key?(item)
425
+
426
+ seen[item] = true
427
+ end
428
+ named
429
+ end
430
+
431
+ # Whether the orders and positions are exact under the rule: no free item barred from leading, and at most
432
+ # EXACT_ORDER_MAX free items.
433
+ def order_exact?(rule, free)
434
+ free.size <= EXACT_ORDER_MAX && contenders(rule[:may_lead], free).size == free.size
435
+ end
436
+
437
+ # The free items `named` holds, in its order (each free rank in turn, on the free slots of the free-slot rule,
438
+ # `rank_free_slots`), or nil when the order cannot be the rule's: a pinned slot naming another item, or a pinned
439
+ # item on a free slot.
440
+ def order_sequence(named, pinned)
441
+ return named if pinned.empty?
442
+ return nil if pinned.any? { |slot, item| slot < named.size && named[slot] != item }
443
+
444
+ held = pinned.values
445
+ sequence = rank_free_slots(pinned).take_while { |slot| slot < named.size }.map { |slot| named[slot] }
446
+ sequence.any? { |item| held.include?(item) } ? nil : sequence
447
+ end
448
+
449
+ # P(the free ranks begin with `sequence`) under the rule, exact (no item barred from leading): the shuffled,
450
+ # explored and drawn parts by the shares the pick odds read (`odds_shares`).
451
+ def order_exact(sequence, free, rule)
452
+ return 1.0 if sequence.empty?
453
+
454
+ grid = order_grid(free)
455
+ eligible, explored, drawn = odds_shares(free, rule)
456
+ size = free.size
457
+ probability = 0.0
458
+ if rule[:shuffle].positive?
459
+ arrangements = ((size - sequence.size + 1)..size).reduce(1.0) { |product, count| product * count }
460
+ probability += rule[:shuffle] / arrangements
461
+ end
462
+ if explored.positive? && eligible.key?(sequence.first)
463
+ probability += explored / eligible.size * order_chain(sequence.drop(1), free - sequence.first(1), grid)
464
+ end
465
+ probability += drawn * order_chain(sequence, free, grid) if drawn.positive?
466
+ odds_bounded(probability)
467
+ end
468
+
469
+ # T(sequence among `among`) = P(X_s1 > ... > X_sm > every other item of `among`), on the grid: 1 for no item; an
470
+ # order's last item, when no other is left, is the rest; a chain of one item is its P(largest) (the set's one
471
+ # P(largest) by kind); longer chains are the Stieltjes sum on both levels of the grid, each level on its own
472
+ # item's points, reading the level below where it is held (`order_read`), extrapolated.
473
+ def order_chain(sequence, among, grid)
474
+ rest = among - sequence
475
+ if rest.empty?
476
+ rest = sequence.last(1)
477
+ sequence = sequence[0...-1]
478
+ end
479
+ return 1.0 if sequence.empty?
480
+ return order_largest(among)[among.index(sequence.first)] if sequence.size == 1
481
+
482
+ fine, coarse = grid.map do |frames|
483
+ deepest = frames.fetch(sequence.last)
484
+ below = order_product(rest.map { |item| deepest.fetch(item) })
485
+ (sequence.size - 1).downto(1) do |rank|
486
+ item = sequence[rank]
487
+ held = order_step(frames.fetch(item).fetch(item), below)
488
+ below = order_read(frames, item, held, sequence[rank - 1])
489
+ end
490
+ head = sequence.first
491
+ order_total(frames.fetch(head).fetch(head), below)
492
+ end
493
+ odds_bounded(order_extrapolated(fine, coarse))
494
+ end
495
+
496
+ # H of `item`, held at the item's own points (`held`), read at the points of `reader`: as it stands where both
497
+ # share one grid (Betas); else at each of the reader's points mapped into the item's own z, where the grid holds
498
+ # the item's distribution function F at the reader's points (Phi(((the reader's mean - the item's mean) + the
499
+ # reader's sd * z) / the item's sd), the means' difference formed first), linearly in F between the two of the
500
+ # item's own points around it (`order_interpolated`).
501
+ def order_read(frames, item, held, reader)
502
+ own = frames.fetch(item)
503
+ at = frames.fetch(reader)
504
+ return held if own.equal?(at)
505
+
506
+ order_interpolated(own.fetch(item), held, at.fetch(item))
507
+ end
508
+
509
+ # `held` (H at the points where F reads `levels`, both in the points' order, F nondecreasing) read at each of
510
+ # `wanted` (F at the points read), linearly in F between the two points around it: the first point whose F is at
511
+ # least the one wanted, and the one before it. Monotone, and exact at a point whose F is one of `levels`: between
512
+ # two points H moves by the increment of F times a mean of what it integrates, so where F does not move H does not
513
+ # either, and any point of a run of equal F reads one H.
514
+ def order_interpolated(levels, held, wanted)
515
+ size = levels.size
516
+ index = 0
517
+ wanted.map do |level|
518
+ index -= 1 while index.positive? && levels[index - 1] >= level
519
+ index += 1 while index < size - 1 && levels[index] < level
520
+ next held[index] if index.zero? || levels[index] <= level
521
+
522
+ low = levels[index - 1]
523
+ held[index - 1] + ((held[index] - held[index - 1]) * ((level - low) / (levels[index] - low)))
524
+ end
525
+ end
526
+
527
+ # The Stieltjes sum extrapolated from the grid (`fine`) and its every other level (`coarse`): fine + (fine -
528
+ # coarse) / 3, which takes out the trapezoid's error that falls as the square of the spacing (Richardson).
529
+ def order_extrapolated(fine, coarse)
530
+ fine + ((fine - coarse) / 3.0)
531
+ end
532
+
533
+ # The pointwise product of the columns (each item's distribution function on the grid).
534
+ def order_product(columns)
535
+ return columns.first if columns.size == 1
536
+
537
+ Array.new(columns.first.size) do |point|
538
+ columns.inject(1.0) { |product, column| product * column[point] }
539
+ end
540
+ end
541
+
542
+ # THE trapezoid's Stieltjes increment, written once (`order_step`, `order_total` and `order_accumulate` sum it):
543
+ # the integrator's increment over an interval times the mean of the integrand at the interval's two ends.
544
+ def order_trapezoid(increment, before, now)
545
+ increment * (before + now) * 0.5
546
+ end
547
+
548
+ # One Stieltjes step: H_k at every grid point, from its item's distribution function `own` and H_(k+1) `below`,
549
+ # 0 at the first point.
550
+ def order_step(own, below)
551
+ size = own.size
552
+ step = Array.new(size, 0.0)
553
+ point = 1
554
+ while point < size
555
+ step[point] = step[point - 1] + order_trapezoid(own[point] - own[point - 1], below[point - 1], below[point])
556
+ point += 1
557
+ end
558
+ step
559
+ end
560
+
561
+ # The last Stieltjes step's value at the top of the grid.
562
+ def order_total(own, below)
563
+ order_step(own, below).last
564
+ end
565
+
566
+ # { item => [P(item at free rank l), l = 0 ... n - 1] } for the free items under the rule, exact (no item barred
567
+ # from leading): the shuffled part 1/n at every rank, the explored and drawn parts by rank (above).
568
+ def order_ranks(free, rule)
569
+ return {} if free.empty?
570
+ return { free.first => [ 1.0 ] } if free.size == 1
571
+
572
+ size = free.size
573
+ eligible, explored, drawn = odds_shares(free, rule)
574
+ grid = order_grid(free)
575
+ largest = order_largest(free)
576
+ sums = order_coefficient_sums(free, grid, eligible, explored.positive?)
577
+ free.each_with_index.to_h do |item, index|
578
+ by_draw, by_explore = sums.fetch(item)
579
+ row = Array.new(size) do |rank|
580
+ probability = rule[:shuffle] / size
581
+ probability += drawn * (rank.zero? ? largest[index] : by_draw[rank]) if drawn.positive?
582
+ if explored.positive?
583
+ probability += explored * order_explored(item, rank, free, eligible, by_explore) / eligible.size
584
+ end
585
+ odds_bounded(probability)
586
+ end
587
+ [ item, row ]
588
+ end
589
+ end
590
+
591
+ # |E| times the explored part's P(item at `rank`): 1 at rank 0 for an eligible item; at rank 1 the sum over the
592
+ # eligible h != item of P(item is the largest among the free items less h), the set's one P(largest); deeper, the
593
+ # Stieltjes sum of the coefficient over those h.
594
+ def order_explored(item, rank, free, eligible, by_explore)
595
+ return eligible.key?(item) ? 1.0 : 0.0 if rank.zero?
596
+ return by_explore[rank - 1] unless rank == 1
597
+
598
+ eligible.each_key.sum do |head|
599
+ next 0.0 if head == item
600
+
601
+ among = free - [ head ]
602
+ order_largest(among)[among.index(item)]
603
+ end
604
+ end
605
+
606
+ # { item => [the Stieltjes sums by draw, one a rank, and by explore, one a rank less one] }. At every one of the
607
+ # item's own points, the coefficients of the product over the other free items j of (F_j + z (1 - F_j)) (by draw)
608
+ # and, when `explored`, of the sum over the eligible h of that product without h (by explore); each increment of
609
+ # the item's own F times the mean of the coefficients at its two ends; on both levels of the grid, extrapolated
610
+ # (`order_extrapolated`).
611
+ def order_coefficient_sums(free, grid, eligible, explored)
612
+ free.to_h do |item|
613
+ others = free.reject { |other| other == item }
614
+ flags = others.map { |other| eligible.key?(other) }
615
+ fine, coarse = grid.map do |frames|
616
+ columns = frames.fetch(item)
617
+ order_item_sums(columns.fetch(item), others.map { |other| columns.fetch(other) }, flags, explored)
618
+ end
619
+ sums = fine.zip(coarse).map do |by_fine, by_coarse|
620
+ by_fine.zip(by_coarse).map { |sum, half| order_extrapolated(sum, half) }
621
+ end
622
+ [ item, sums ]
623
+ end
624
+ end
625
+
626
+ # One item's sums (`order_coefficient_sums`): `own` its distribution function on its points, `columns` the other
627
+ # items' there, `flags` whether each other is eligible. The products are built factor by factor in place, highest
628
+ # coefficient first, the explored sum before the product it reads (B' = B (f + c z) + [eligible] A,
629
+ # A' = A (f + c z)).
630
+ def order_item_sums(own, columns, flags, explored)
631
+ size = columns.size + 1
632
+ product = Array.new(size, 0.0)
633
+ product_before = Array.new(size, 0.0)
634
+ without = Array.new(size, 0.0)
635
+ without_before = Array.new(size, 0.0)
636
+ by_draw = Array.new(size, 0.0)
637
+ by_explore = Array.new(size, 0.0)
638
+ own.each_index do |point|
639
+ product.fill(0.0)
640
+ product[0] = 1.0
641
+ without.fill(0.0) if explored
642
+ columns.each_with_index do |column, factor|
643
+ below = column[point]
644
+ above = 1.0 - below
645
+ order_add_factor(without, product, factor, below, above, flags[factor]) if explored
646
+ order_multiply(product, factor, below, above)
647
+ end
648
+ if point.positive?
649
+ increment = own[point] - own[point - 1]
650
+ unless increment.zero?
651
+ order_accumulate(by_draw, increment, product_before, product)
652
+ order_accumulate(by_explore, increment, without_before, without) if explored
653
+ end
654
+ end
655
+ product, product_before = product_before, product
656
+ without, without_before = without_before, without
657
+ end
658
+ [ by_draw, by_explore ]
659
+ end
660
+
661
+ # `coefficients` (degree `factor`, held in 0..factor) times (below + above z), in place.
662
+ def order_multiply(coefficients, factor, below, above)
663
+ coefficients[factor + 1] = above * coefficients[factor]
664
+ degree = factor
665
+ while degree.positive?
666
+ coefficients[degree] = (below * coefficients[degree]) + (above * coefficients[degree - 1])
667
+ degree -= 1
668
+ end
669
+ coefficients[0] *= below
670
+ end
671
+
672
+ # The explored sum B (degree factor - 1) times (below + above z), plus the product A as it stands when the
673
+ # factor's item is eligible, in place.
674
+ def order_add_factor(sum, product, factor, below, above, eligible)
675
+ degree = factor
676
+ while degree >= 0
677
+ value = below * sum[degree]
678
+ value += above * sum[degree - 1] if degree.positive?
679
+ value += product[degree] if eligible
680
+ sum[degree] = value
681
+ degree -= 1
682
+ end
683
+ end
684
+
685
+ # Each coefficient's sum plus its trapezoid increment (`order_trapezoid`) over the interval.
686
+ def order_accumulate(sums, increment, before, now)
687
+ index = 0
688
+ size = sums.size
689
+ while index < size
690
+ sums[index] += order_trapezoid(increment, before[index], now[index])
691
+ index += 1
692
+ end
693
+ end
694
+
695
+ # The rank table placed on the slots: each free rank on the next free slot (the free-slot rule,
696
+ # `rank_free_slots`), a pinned item 1 at its own slot, every item in item order, a row a slot.
697
+ def order_slotted(ranks, free, pinned)
698
+ slots = rank_free_slots(pinned)
699
+ table = @items.to_h { |item| [ item, Array.new(@items.size, 0.0) ] }
700
+ pinned.each { |slot, item| table.fetch(item)[slot] = 1.0 }
701
+ free.each { |item| ranks.fetch(item).each_with_index { |probability, rank| table.fetch(item)[slots[rank]] = probability } }
702
+ table
703
+ end
704
+
705
+ # The grid of the free items' order odds, built once a set and kept, on two levels: [fine, coarse], each { item
706
+ # => { free item => its distribution function F at each of the item's points } }, the coarse level a part of the
707
+ # fine one's points: the infinities, the quantiles at every other level (2/ORDER_LEVELS, 4/ORDER_LEVELS, ...)
708
+ # and, for Normals, the levels 0 and 1 and the points beside each item's carried points. Betas share one grid
709
+ # (`order_beta_grid`), so every item names one Hash; Normals have each item's own (`order_normal_grid`). F is 0
710
+ # at -infinity and 1 at +infinity. The free items' P(largest) is read first, the one guard of what the integrals
711
+ # can read.
712
+ def order_grid(free)
713
+ order_memo(:grid, free) do
714
+ order_largest(free)
715
+ @kind == :beta ? order_beta_grid(free) : order_normal_grid(free)
716
+ end
717
+ end
718
+
719
+ # The Betas' one grid, in the position the rule compares their draws by (BetaArgmax.position's log scale,
720
+ # ranking.rb): -infinity, +infinity and every free item's position at the ORDER_LEVELS - 1 levels inside (0, 1)
721
+ # on its scaled pair (where a draw at that uniform compares), each item's F read there
722
+ # (`order_beta_distribution`).
723
+ def order_beta_grid(free)
724
+ tagged = [ [ -Float::INFINITY, true ], [ Float::INFINITY, true ] ]
725
+ free.each do |item|
726
+ alpha, beta = @parameters.fetch(item)
727
+ (1...ORDER_LEVELS).each do |level|
728
+ tagged << [ BetaArgmax.position(level.fdiv(ORDER_LEVELS), alpha, beta), level.even? ]
729
+ end
730
+ end
731
+ points, coarse = order_merged(tagged)
732
+ fine = free.to_h do |item|
733
+ alpha, beta = @parameters.fetch(item)
734
+ [ item, points.map { |position| order_beta_distribution(position, alpha, beta) }.freeze ]
735
+ end.freeze
736
+ thinned = fine.transform_values { |column| column.values_at(*coarse).freeze }.freeze
737
+ [ free.to_h { |item| [ item, fine ] }.freeze, free.to_h { |item| [ item, thinned ] }.freeze ].freeze
738
+ end
739
+
740
+ # Each Normal's own points, in its own standardised variable z = (x - mean) / sd on its scaled pair (as
741
+ # NormalArgmax.probabilities integrates each member): -infinity, +infinity, and every free item's quantile at
742
+ # each level 0, 1/ORDER_LEVELS, ..., 1 (the standard quantile at the level read as a quantile draw's uniform is,
743
+ # Draw.open_uniform, so the levels 0 and 1 read the open grid's ends, 2**-53 within them) carried into z, ((the
744
+ # other's mean - the item's mean) + the other's sd * the standard quantile) / the item's sd, the means'
745
+ # difference formed first, where it is finite; beside each item's carried points the second Float below the
746
+ # lowest and above the highest, so an item whose points round together in z rises wholly between two points. An
747
+ # item's own ends keep its mass between -infinity and its first quantile at 2**-53, where the other items' F
748
+ # reads 0 by the formula whatever their scale. Every free item's F is read at those points
749
+ # (`order_normal_column`).
750
+ def order_normal_grid(free)
751
+ scores = (0..ORDER_LEVELS).map do |level|
752
+ Significance.normal_quantile(Draw.open_uniform(level.fdiv(ORDER_LEVELS)))
753
+ end
754
+ own = free.map do |item|
755
+ mean, deviation = @parameters.fetch(item)
756
+ tagged = [ [ -Float::INFINITY, true ], [ Float::INFINITY, true ] ]
757
+ free.each { |other| order_carried(tagged, scores, mean, deviation, @parameters.fetch(other)) }
758
+ points, coarse = order_merged(tagged)
759
+ columns = free.to_h do |other|
760
+ [ other, order_normal_column(points, mean, deviation, @parameters.fetch(other)).freeze ]
761
+ end
762
+ [ columns.freeze, columns.transform_values { |column| column.values_at(*coarse).freeze }.freeze ]
763
+ end
764
+ [ free.zip(own.map(&:first)).to_h.freeze, free.zip(own.map(&:last)).to_h.freeze ].freeze
765
+ end
766
+
767
+ # The other item's quantiles carried into the item's z (`order_normal_grid`) by the one map between two Normals'
768
+ # standardised variables (Standardised.carried, the means' difference formed first), each tagged coarse at every
769
+ # other level (the ends included), and the two points beside them, tagged coarse, added to `tagged`.
770
+ def order_carried(tagged, scores, mean, deviation, (other_mean, other_deviation))
771
+ lowest = Float::INFINITY
772
+ highest = -Float::INFINITY
773
+ scores.each_with_index do |score, index|
774
+ z = Standardised.carried(other_mean, other_deviation, score, mean, deviation)
775
+ next unless z.finite?
776
+
777
+ tagged << [ z, index.even? ]
778
+ lowest = z if z < lowest
779
+ highest = z if z > highest
780
+ end
781
+ return unless lowest.finite?
782
+
783
+ tagged << [ lowest.prev_float.prev_float, true ] << [ highest.next_float.next_float, true ]
784
+ end
785
+
786
+ # The other item's F at each of the item's points z: Phi(((the item's mean - the other's mean) + the item's sd *
787
+ # z) / the other's sd), the item's z carried into the other's by the one map (Standardised.carried, the means'
788
+ # difference formed first), so a finite numerator over a standard deviation reads 0 or 1 where the quotient
789
+ # leaves a Float, never a NaN (0 at -infinity, 1 at +infinity).
790
+ def order_normal_column(points, mean, deviation, (other_mean, other_deviation))
791
+ points.map do |z|
792
+ Significance.normal_cdf(Standardised.carried(mean, deviation, z, other_mean, other_deviation))
793
+ end
794
+ end
795
+
796
+ # [the points, sorted, each once; the indices of the coarse ones] from [point, coarse?] pairs: a point given more
797
+ # than once is coarse when any of its givings is.
798
+ def order_merged(tagged)
799
+ points = []
800
+ coarse = []
801
+ tagged.sort_by(&:first).each do |point, half|
802
+ if points.empty? || points.last != point
803
+ points << point
804
+ coarse << half
805
+ elsif half
806
+ coarse[-1] = true
807
+ end
808
+ end
809
+ [ points.freeze, coarse.each_index.select { |index| coarse[index] }.freeze ]
810
+ end
811
+
812
+ # Beta(alpha, beta)'s distribution function at the position p (x = exp(p + JOINT) below 0, else 1 -
813
+ # exp(JOINT - p): the position's two halves, `rank_beta_half`, as the quantile draws read them), read in each
814
+ # half from its own end, as BetaArgmax reads its two halves: below 0 the item's own at log x = p + JOINT, from 0
815
+ # up 1 less the reflected item's (beta, alpha) at log (1 - x) = JOINT - p (`order_beta_end`). So no mass near
816
+ # either end rounds into the end itself.
817
+ def order_beta_distribution(position, alpha, beta)
818
+ lower, log_end = rank_beta_half(position)
819
+ return order_beta_end(log_end, alpha, beta) if lower
820
+
821
+ 1.0 - order_beta_end(log_end, beta, alpha)
822
+ end
823
+
824
+ # I_w(p, q) at log w = v (v at most JOINT, -infinity included): BetaArgmax.distribution at w = exp(v) where w is
825
+ # at least ORDER_FLOOR, the function BetaArgmax integrates and positions on. Below it, where a Float holds fewer
826
+ # digits of w and then none, the distribution function is its leading term w**p / (p * B(p, q)) to a Float's
827
+ # precision (the next term is about q * w of it, below 1e-301 within the scaled pairs' total), a power of w:
828
+ # BetaArgmax.distribution at ORDER_FLOOR times exp(p * (v - log ORDER_FLOOR)), a factor at most 1.
829
+ def order_beta_end(v, p, q)
830
+ return BetaArgmax.distribution(Math.exp(v), p, q) if v >= ORDER_LOG_FLOOR
831
+
832
+ BetaArgmax.distribution(ORDER_FLOOR, p, q) * Math.exp(p * (v - ORDER_LOG_FLOOR))
833
+ end
834
+
835
+ # P(largest) of each item of `among` (in its order), the set's one P(largest) by kind, kept once read.
836
+ def order_largest(among)
837
+ order_memo(:largest, among) { largest_draw(among).first.freeze }
838
+ end
839
+
840
+ # The value kept on this set under `what` for the items `among` (a frozen copy of the list), computed by the
841
+ # block once (a set the caller froze keeps nothing and computes each time).
842
+ def order_memo(what, among)
843
+ return yield if frozen?
844
+
845
+ kept = (@order_kept ||= {})
846
+ key = [ what, among.frozen? ? among : among.dup.freeze ].freeze
847
+ kept.fetch(key) { kept[key] = yield }
848
+ end
849
+
850
+ # The rule's visits from the RNG source: `draws` whole orders (the order rule's own ordering, as `rank(rng:)`
851
+ # orders, pinned items at their slots, its Sampler draws on the scaled pairs, `parameters`, the pairs the exact
852
+ # order odds read, compared as the rule compares them) drawn on Random.new(seed), each yielded.
853
+ def order_visits(rule, draws, seed)
854
+ rng = Random.new(seed)
855
+ draws.times { yield rank_ordered(rule, nil, nil, rng).first }
856
+ end
857
+
858
+ # [the share of the rule's visits whose whole order begins with `named`, its standard error].
859
+ def order_counted(named, rule, draws, seed)
860
+ length = named.size
861
+ hits = 0
862
+ order_visits(rule, draws, seed) { |whole| hits += 1 if whole.first(length) == named }
863
+ share = hits.fdiv(draws)
864
+ [ share, order_binomial_error(share, draws) ]
865
+ end
866
+
867
+ # [{ item => [the share of visits with the item at each slot] }, the largest standard error of those shares].
868
+ def order_counted_positions(rule, draws, seed)
869
+ size = @items.size
870
+ counts = @items.to_h { |item| [ item, Array.new(size, 0) ] }
871
+ order_visits(rule, draws, seed) do |whole|
872
+ whole.each_with_index { |item, slot| counts.fetch(item)[slot] += 1 }
873
+ end
874
+ error = 0.0
875
+ table = counts.transform_values do |row|
876
+ row.map do |count|
877
+ share = count.fdiv(draws)
878
+ error = [ error, order_binomial_error(share, draws) ].max
879
+ share
880
+ end
881
+ end
882
+ [ table, error ]
883
+ end
884
+ end
885
+
886
+ # What a Choice or an Order carries from the set that made it (ranking.rb sets it before the freeze): [the set,
887
+ # the rule, the cache]; an ArgumentError for one made by hand, which carries none.
888
+ module OrderCarried
889
+ module_function
890
+
891
+ def read(answer)
892
+ posteriors = answer.instance_variable_get(:@posteriors)
893
+ rule = answer.instance_variable_get(:@rule)
894
+ cache = answer.instance_variable_get(:@cache)
895
+ return [ posteriors, rule, cache ] if Posteriors === posteriors && Hash === rule && Hash === cache
896
+
897
+ raise ArgumentError, "a #{Inputs.class_name(answer)} answers its probability only when a posterior set made " \
898
+ "it (it carries the set, the rule and the cache)"
899
+ end
900
+ end
901
+ private_constant :OrderCarried
902
+
903
+ # A pick's odds under the rule that made it (probabilities.rb).
904
+ class Choice
905
+ # P(head = this head) under the rule the pick was made by: its set's `pick_probabilities` of the head (at the
906
+ # default draws and seed where they are a Monte Carlo), 1.0 for no head (an empty set). Computed once and kept
907
+ # in the cache it carries.
908
+ def probability
909
+ posteriors, rule, cache = OrderCarried.read(self)
910
+ cache.fetch(:probability) do
911
+ cache[:probability] = head.nil? ? 1.0 : posteriors.pick_probabilities(**rule)[:heads].fetch(head)
912
+ end
913
+ end
914
+ end
915
+
916
+ # An order's odds under the rule that made it (probabilities.rb).
917
+ class Order
918
+ # P(the rule's order begins with these items) under the rule the order was made by: its set's
919
+ # `order_probability` of its items (at the default draws and seed where they are a Monte Carlo). Computed once
920
+ # and kept in the cache it carries.
921
+ def probability
922
+ posteriors, rule, cache = OrderCarried.read(self)
923
+ cache.fetch(:probability) do
924
+ cache[:probability] = posteriors.order_probability(items, **rule)[:probability]
925
+ end
926
+ end
927
+ end
928
+ end
929
+ end