skiftet_statistical 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +308 -8
- data/README.md +1416 -39
- data/lib/skiftet_statistical/additive_contrasts.rb +894 -0
- data/lib/skiftet_statistical/bandit/allocator.rb +477 -63
- data/lib/skiftet_statistical/bandit/checks.rb +128 -0
- data/lib/skiftet_statistical/bandit/decay.rb +86 -0
- data/lib/skiftet_statistical/bandit/draw.rb +179 -0
- data/lib/skiftet_statistical/bandit/normal_argmax.rb +209 -0
- data/lib/skiftet_statistical/bandit/pass.rb +94 -0
- data/lib/skiftet_statistical/bandit/posteriors.rb +497 -0
- data/lib/skiftet_statistical/bandit/prior.rb +264 -0
- data/lib/skiftet_statistical/bandit/probabilities.rb +929 -0
- data/lib/skiftet_statistical/bandit/ranking.rb +552 -0
- data/lib/skiftet_statistical/bandit/summary.rb +152 -0
- data/lib/skiftet_statistical/bandit/tallies.rb +297 -0
- data/lib/skiftet_statistical/bandit.rb +3 -3
- data/lib/skiftet_statistical/beta_argmax.rb +526 -0
- data/lib/skiftet_statistical/beta_function.rb +28 -1
- data/lib/skiftet_statistical/cholesky.rb +244 -0
- data/lib/skiftet_statistical/cusum.rb +183 -0
- data/lib/skiftet_statistical/descriptive.rb +3 -3
- data/lib/skiftet_statistical/gauss_legendre.rb +73 -0
- data/lib/skiftet_statistical/hierarchical_rates.rb +450 -0
- data/lib/skiftet_statistical/inputs.rb +214 -0
- data/lib/skiftet_statistical/linear_value.rb +364 -0
- data/lib/skiftet_statistical/numbers.rb +98 -0
- data/lib/skiftet_statistical/prequential.rb +206 -0
- data/lib/skiftet_statistical/shared_credit.rb +811 -0
- data/lib/skiftet_statistical/significance.rb +5 -6
- data/lib/skiftet_statistical/version.rb +1 -1
- data/lib/skiftet_statistical.rb +43 -4
- metadata +37 -11
|
@@ -0,0 +1,929 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../numbers"
|
|
4
|
+
require_relative "../inputs"
|
|
5
|
+
require_relative "../beta_argmax"
|
|
6
|
+
require_relative "../significance"
|
|
7
|
+
require_relative "checks"
|
|
8
|
+
require_relative "draw"
|
|
9
|
+
require_relative "normal_argmax"
|
|
10
|
+
require_relative "posteriors"
|
|
11
|
+
require_relative "ranking"
|
|
12
|
+
|
|
13
|
+
module SkiftetStatistical
|
|
14
|
+
class Bandit
|
|
15
|
+
# The posterior set's odds: the probability of each head a pick answers, of each order and of each item at each
|
|
16
|
+
# position that `rank` answers, all under the rule that made the draw, and the probability that a two-layer draw
|
|
17
|
+
# shows an item.
|
|
18
|
+
#
|
|
19
|
+
# set = SkiftetStatistical::Bandit::Posteriors.beta({ "a" => [ 1.0, 1.0 ], "b" => [ 2.0, 1.0 ] })
|
|
20
|
+
# set.pick_probabilities[:heads] # => { "a" => 1/3, "b" => 2/3 }
|
|
21
|
+
# set.pick_probabilities(explore: 0.1)[:heads] # => { "a" => 0.05 + 0.9 * 1/3, "b" => 0.05 + 0.9 * 2/3 }
|
|
22
|
+
# set.order_probability(%w[b a]) # => { probability: 2/3, method: :exact, standard_error: 0.0 }
|
|
23
|
+
# set.positions[:probabilities] # => { "a" => [1/3, 2/3], "b" => [2/3, 1/3] }
|
|
24
|
+
# set.order_summary(%w[b a]) # => { probability: 2/3, positions: { "a" => [...], ... } }
|
|
25
|
+
# set.rank(key: "k1").probability # => the probability of the order that key drew
|
|
26
|
+
# SkiftetStatistical::Bandit::Posteriors.mixture(
|
|
27
|
+
# shares: { "x" => 0.25, "y" => 0.75 },
|
|
28
|
+
# probabilities: { "x" => { "a" => 0.5, "b" => 0.5 }, "y" => { "a" => 0.2, "c" => 0.8 } }
|
|
29
|
+
# ) # => { "a" => 0.275, "b" => 0.125, "c" => 0.6 }
|
|
30
|
+
#
|
|
31
|
+
# THE PICK (`pick_probabilities`), under the rule `pick` draws by: `shuffle` and `explore` (each in [0, 1], summing
|
|
32
|
+
# to at most 1), `may_lead` (nil: every item) and `min_views` (0). The contenders C are the items `may_lead` names
|
|
33
|
+
# (every item when it is nil or names none), the eligible E the contenders with at least `min_views` views. The
|
|
34
|
+
# head is item c with probability
|
|
35
|
+
#
|
|
36
|
+
# shuffle / |C| [c in C] + explore / |E| [c in E, E not empty] + (1 - shuffle - explore [E not empty]) P(c)
|
|
37
|
+
#
|
|
38
|
+
# with P(c) the probability that c's draw is the largest among C's: BetaArgmax.probabilities or
|
|
39
|
+
# NormalArgmax.probabilities on the scaled pairs (`parameters`), through the set's one P(largest) by kind, up to
|
|
40
|
+
# EXACT_PICK_MAX contenders, each by its quadrature within its stated bound, not to the last digit: BetaArgmax's
|
|
41
|
+
# within TAIL (1e-10) a member, and NormalArgmax's, each member in its own standardised variable, within
|
|
42
|
+
# 2 k (k + 1) TAIL + 2e-12 of the exact probability for k contenders (1.2e-9 for two, 8.4e-8 for 20). Above
|
|
43
|
+
# EXACT_PICK_MAX contenders P(c) is the stated Monte Carlo below, and the rest of the formula stands as it is. When
|
|
44
|
+
# no contender has `min_views` views the exploration share goes back to the draw, as the pick does. Every source of
|
|
45
|
+
# the pick (a key, given uniforms, an RNG) draws exactly from the posteriors on the scaled pairs, and compares two
|
|
46
|
+
# draws as the posteriors order them (a Beta by its position or its value, a Normal by the exact sign of its mean
|
|
47
|
+
# + sd * z against the other's, ranking.rb), so these are, within that bound, the odds of a pick from any of them.
|
|
48
|
+
# The answer:
|
|
49
|
+
# - `heads`: { item => P(head = item) } for every item, in item order (0.0 for an item that may not lead);
|
|
50
|
+
# - `leaders`: the pick's leader is its head when the head has at least `min_views` views, else nil, so each item
|
|
51
|
+
# with at least `min_views` views gets its head's probability and nil gets the heads of the items below it
|
|
52
|
+
# (0.0 when there are none); an empty set answers no heads and a leader nil with probability 1;
|
|
53
|
+
# - `method`: :exact (the quadrature, not a Monte Carlo), and `standard_error`: 0.0; or, above EXACT_PICK_MAX
|
|
54
|
+
# contenders with a share left to the draw, :monte_carlo and the largest standard error of a head or a leader.
|
|
55
|
+
#
|
|
56
|
+
# THE ORDERS (`order_probability(order)`) and THE POSITIONS (`positions`), under the rule `rank` draws by: the
|
|
57
|
+
# pick's four keywords and `pinned` ({ position => item }) and `top`. Pinned items keep their positions; the n free
|
|
58
|
+
# items fill the other positions in order, each free rank on the next free slot. `order` is a list of distinct
|
|
59
|
+
# items, as `rank` answers them (pinned items at their positions): its probability is that of the rule's whole
|
|
60
|
+
# order beginning with it, so for an order of `top` items (every item when top is nil) it is the probability that
|
|
61
|
+
# `rank` answers it, and `top` itself changes no odds. An order whose pinned slot holds another item, or that puts
|
|
62
|
+
# a pinned item on a free slot, has probability 0; the empty order 1. With s1, ..., sm the free items the order
|
|
63
|
+
# names, in its order, its probability is
|
|
64
|
+
#
|
|
65
|
+
# shuffle / (n (n - 1) ... (n - m + 1))
|
|
66
|
+
# + explore [E not empty] [s1 in E] / |E| * T(s2, ..., sm among the free items less s1)
|
|
67
|
+
# + (1 - shuffle - explore [E not empty]) * T(s1, ..., sm among the free items),
|
|
68
|
+
#
|
|
69
|
+
# T(s1, ..., sm among S) = P(X_s1 > ... > X_sm > every other item of S), and `positions` answers P(item at slot)
|
|
70
|
+
# on the same parts: the shuffled part 1/n at every free rank; the explored part 1/|E| at rank 0 for an eligible
|
|
71
|
+
# item, and at rank l >= 1 the sum over h in E, h != i, of 1/|E| * P(i at rank l - 1 by draw among the free items
|
|
72
|
+
# less h); the drawn part P(i at rank l by draw among the free items). A pinned item is 1 at its own slot.
|
|
73
|
+
#
|
|
74
|
+
# Each is exact, by a quadrature, when no free item is barred from leading (`may_lead` names every free item, or
|
|
75
|
+
# none) and there are at most EXACT_ORDER_MAX free items; otherwise it is the stated Monte Carlo below. A rank by
|
|
76
|
+
# draw at the head is P(largest), the set's one P(largest) by kind (above); every other is a Stieltjes sum against
|
|
77
|
+
# exact distribution increments, each level of a chain and each item's position sums integrated over its own
|
|
78
|
+
# item's points, in a coordinate where that item's own points keep their digits (on the scaled pairs). Each item's
|
|
79
|
+
# points are -infinity, +infinity and the free items' quantiles at the ORDER_LEVELS - 1 evenly spaced levels
|
|
80
|
+
# inside (0, 1) (for Normals also at 0 and 1, below), sorted, each once.
|
|
81
|
+
# - Betas share one grid, in the position of BetaArgmax.position (the key ranking.rb compares their draws by:
|
|
82
|
+
# log(2 x) up to 1/2, then -log(2 (1 - x))), where a Beta's points keep their digits. Every item's distribution
|
|
83
|
+
# function is read there by BetaArgmax.distribution (the very function its P(largest) and its draws read), each
|
|
84
|
+
# half from its own end as BetaArgmax reads its halves, and below 2**-1023 from that end, where a Float holds
|
|
85
|
+
# fewer digits of x and then none (and where a parameter near SMALLEST_PARAMETER holds nearly all of its mass),
|
|
86
|
+
# by its leading term, a power of x scaled from BetaArgmax.distribution at 2**-1023.
|
|
87
|
+
# - Normals have no such shared coordinate within the input domain: x itself rounds a narrow item's points
|
|
88
|
+
# together at a large mean, x less the largest mean rounds two narrow items near 0 together beside a far item,
|
|
89
|
+
# and a scale by a standard deviation sends a wide item's points past a Float's range. So each item has its own
|
|
90
|
+
# points in its own standardised variable z = (x - mean) / sd, as NormalArgmax.probabilities integrates each
|
|
91
|
+
# member: every free item's quantiles, at the levels inside (0, 1) and at 0 and 1 read as a quantile draw's
|
|
92
|
+
# uniform is (Draw.open_uniform, 2**-53 within the ends), carried into z as ((the other's mean - the item's mean)
|
|
93
|
+
# + the other's sd * the standard quantile) / the item's sd (the means' difference formed first; a point beyond a
|
|
94
|
+
# Float's range is left out), and beside each item's carried points the second Float below the lowest and above
|
|
95
|
+
# the highest, so that an item whose points round together in z rises wholly between two points. The ends keep
|
|
96
|
+
# an item's own mass below its first quantile on a finite point, where the other items' F hold their values, not
|
|
97
|
+
# the 0 every F reads at -infinity. Every item's distribution function is read at the item's points as
|
|
98
|
+
# Phi(((the item's mean - the other's mean) + the item's sd * z) / the other's sd).
|
|
99
|
+
# So the chain, each level on its own item's points,
|
|
100
|
+
#
|
|
101
|
+
# H(x) = the product of the other items' F (1 when none); then, for k = m down to 1,
|
|
102
|
+
# H_k(x_j) = H_k(x_(j-1)) + (F_sk(x_j) - F_sk(x_(j-1))) * (H_(k+1)(x_(j-1)) + H_(k+1)(x_j)) / 2
|
|
103
|
+
#
|
|
104
|
+
# gives T = H_1 at the top (an order's last item, when no other is left, is the rest, and a chain of one item is
|
|
105
|
+
# its P(largest)). H_(k+1), held on s_(k+1)'s points, is read at s_k's: on the Betas' one grid as it stands; for
|
|
106
|
+
# Normals at each of s_k's points mapped into s_(k+1)'s z (the means' difference formed first, then plus s_k's sd
|
|
107
|
+
# times z, over s_(k+1)'s sd), linearly in F_s(k+1) between the two of s_(k+1)'s points around it, which keeps
|
|
108
|
+
# it monotone and exact where the point is one of s_(k+1)'s (H rises between two points by the increment of F
|
|
109
|
+
# times a mean of what it integrates, so where F does not move H does not either). P(i at rank l by draw) sums each
|
|
110
|
+
# increment of F_i on i's points times the mean, at the interval's two ends, of the z**l coefficient of the
|
|
111
|
+
# product over j != i of (F_j + z (1 - F_j)) (the explored part the same over the items less h, summed over h).
|
|
112
|
+
# Each sum is taken on the points and on their coarse part (the infinities, the quantiles at every other level and,
|
|
113
|
+
# for Normals, the points beside each item's carried points) and extrapolated, fine + (fine - coarse) / 3
|
|
114
|
+
# (Richardson), which takes out the trapezoid's error that falls as the square of the spacing. Each item's mass is
|
|
115
|
+
# split evenly between its own quantiles, so no item is sampled coarsely, a parameter below 1 included, where a
|
|
116
|
+
# trapezoid of the density reads up to 7e-3 off: at 400 levels an item the orders and positions read within 4e-6
|
|
117
|
+
# of the closed forms of Beta(a, 1) and Beta(1, b) items, which race as Plackett-Luce (800 random sets of three and
|
|
118
|
+
# four, a and b from 1e-6 to 10), three identical items every order 1/n! and every position 1/n within 1e-10 at
|
|
119
|
+
# any scale, two narrow Normals beside a far one (deviations 1e-12 to 1e-7, near 0 or 1e9, the far one 1e6 ahead
|
|
120
|
+
# with a deviation of 1 or 1e10) within 1.2e-6 of their closed forms, random Normal sets of three to five (means
|
|
121
|
+
# up to 5e7 either side of 0, deviations 1e-6 to 1e6) with their positions' rows and columns and their orders by
|
|
122
|
+
# head within 3e-6 of 1 and of the pick's odds, and all of them within 1e-4 of the exact figures (the library's
|
|
123
|
+
# stated tolerance), answered as Floats in [0, 1] (`odds_bounded`, the odds' one clamp).
|
|
124
|
+
#
|
|
125
|
+
# THE MONTE CARLO, stated, wherever an odds call has no exact rule (`draws` and `seed` read first, by the bandit's
|
|
126
|
+
# one reading of a whole number, Checks.integer: draws at least 1, seed any Integer within a Float's range):
|
|
127
|
+
# `draws` draws from the RNG source on Random.new(seed), so the same call answers the same figures, each on the
|
|
128
|
+
# scaled pairs (`parameters`), the one pair the exact odds and every draw (`rank`, `pick` and `values`, from any
|
|
129
|
+
# source) read. For the pick, P(c is C's largest draw) is the share of those draws in which c's draw (one Sampler
|
|
130
|
+
# draw per contender, in item order, the set's one RNG draw) is the largest by the order rule's own choice of a
|
|
131
|
+
# first largest, compared by the key the rule compares (a Normal's exact mean + sd * z); the standard error of a
|
|
132
|
+
# head is its drawn share times sqrt(q (1 - q) / draws). For orders and positions, the rule itself visits `draws`
|
|
133
|
+
# times (the order rule's own ordering, as `rank(rng:)` orders), and each figure is the share of visits that answer
|
|
134
|
+
# it, its standard error sqrt(p (1 - p) / draws). The answer's
|
|
135
|
+
# `method` is :monte_carlo and its `standard_error` the largest of its figures'. A Beta parameter far below 1 draws
|
|
136
|
+
# 0.0 in a share of Sampler's draws (its Gamma draw underflows: about a tenth of them at 0.003), where two such
|
|
137
|
+
# draws tie and the tie goes to the earlier item, so below about 0.003 the Monte Carlo reads that tie rule as well
|
|
138
|
+
# as the posterior. It refuses what the exact odds refuse (below), at every size.
|
|
139
|
+
#
|
|
140
|
+
# THE POSTERIOR ORDER (`order_summary(order)`): the posterior alone, no rule: P(X_s1 > ... > X_sm > every other
|
|
141
|
+
# item), that the order is the best one (T above, over every item), and every item's P(at each position) by draw,
|
|
142
|
+
# exact as above, for a set of at most EXACT_ORDER_MAX items.
|
|
143
|
+
#
|
|
144
|
+
# Choice#probability and Order#probability answer the odds of the head or the order they hold under the rule they
|
|
145
|
+
# were made by (the set, the rule and the cache each carries): `pick_probabilities` of the head (1.0 for no head,
|
|
146
|
+
# an empty set) and `order_probability` of the items, at the default draws and seed where those are a Monte Carlo;
|
|
147
|
+
# computed on the first call and kept.
|
|
148
|
+
#
|
|
149
|
+
# Refused, as ArgumentErrors before anything is computed: `draws` that is not an Integer of at least 1, or `seed`
|
|
150
|
+
# that is not an Integer, within a Float's range; a rule `rank` or `pick` refuses (read by the order rule's one
|
|
151
|
+
# reader); an `order` that is not a list of items or that names an item twice; `order_summary` of more than
|
|
152
|
+
# EXACT_ORDER_MAX items; and, by the set's one refusal, a Beta parameter below BetaArgmax::SMALLEST_PARAMETER on the
|
|
153
|
+
# scaled pairs of any free item (one that may not lead included; for the pick every item), the scope a keyed draw
|
|
154
|
+
# of the same set and rule refuses, at every size: the exact odds and the stated Monte Carlo alike (the pick at any
|
|
155
|
+
# number of contenders, the orders and positions with or without items barred from leading, the posterior order);
|
|
156
|
+
# a parameter at it is taken.
|
|
157
|
+
#
|
|
158
|
+
# THE MIXTURE (`Posteriors.mixture`): a first layer draws branch b with probability shares[b], then a second layer
|
|
159
|
+
# shows item c with probability probabilities[b][c]. Item c is shown with probability
|
|
160
|
+
#
|
|
161
|
+
# sum over b of shares[b] * probabilities[b][c]
|
|
162
|
+
#
|
|
163
|
+
# (an item a branch does not name counts 0 there), answered as { item => Float } in the order the items are first
|
|
164
|
+
# named, branch by branch. Branches and items may be any keys a Hash can look up (the library's one test of a name,
|
|
165
|
+
# Inputs.name?), read as given (Symbols and Strings are different keys). Every branch of `shares` must have its
|
|
166
|
+
# probabilities, a Hash (empty for a branch that shows nothing), and `probabilities` may name no branch that
|
|
167
|
+
# `shares` does not; every share and probability is a number >= 0 (by the library's one number reader, in the input
|
|
168
|
+
# domain, Numbers::LARGEST), so no product or sum leaves a Float. An ArgumentError otherwise. Every answer passes
|
|
169
|
+
# the bandit's one check of an answer (Checks.finite_answer).
|
|
170
|
+
class Posteriors
|
|
171
|
+
# The quantile levels an item, evenly spaced, at which the free items split the order odds' grid.
|
|
172
|
+
ORDER_LEVELS = 400
|
|
173
|
+
# Half a Float's smallest normal, 2**-1023: at and below it a Beta's distribution function is its leading term.
|
|
174
|
+
ORDER_FLOOR = 2.0**-1023
|
|
175
|
+
ORDER_LOG_FLOOR = Math.log(ORDER_FLOOR)
|
|
176
|
+
# What the odds compute, exact or by their stated Monte Carlo, and the posterior order, in the words of GB6's
|
|
177
|
+
# refusal: one refusal at every size, over the items a keyed draw of the same set and rule refuses.
|
|
178
|
+
ODDS_PICK_EXACT = "the exact pick odds on the scaled pairs"
|
|
179
|
+
ODDS_PICK_COUNTED = "the pick odds' stated Monte Carlo, which refuses what the exact pick odds on the scaled " \
|
|
180
|
+
"pairs refuse"
|
|
181
|
+
ORDER_EXACT = "the exact order and position odds on the scaled pairs"
|
|
182
|
+
ORDER_COUNTED = "the order and position odds' stated Monte Carlo, which refuses what the exact order and " \
|
|
183
|
+
"position odds on the scaled pairs refuse"
|
|
184
|
+
ORDER_POSTERIOR = "the posterior order and positions on the scaled pairs"
|
|
185
|
+
private_constant :ORDER_LEVELS, :ORDER_FLOOR, :ORDER_LOG_FLOOR, :ODDS_PICK_EXACT, :ODDS_PICK_COUNTED,
|
|
186
|
+
:ORDER_EXACT, :ORDER_COUNTED, :ORDER_POSTERIOR
|
|
187
|
+
|
|
188
|
+
class << self
|
|
189
|
+
# The probability that a two-layer draw shows each item: { item => sum over branches b of shares[b] *
|
|
190
|
+
# probabilities[b][item] }.
|
|
191
|
+
def mixture(shares:, probabilities:)
|
|
192
|
+
odds_refuse_unpaired(shares, probabilities)
|
|
193
|
+
shown = {}
|
|
194
|
+
Inputs.pairs(shares).each do |branch, given_share|
|
|
195
|
+
share = Checks.nonnegative(given_share, "shares[#{Inputs.shown(branch)}]")
|
|
196
|
+
label = "probabilities[#{Inputs.shown(branch)}]"
|
|
197
|
+
Inputs.pairs(Checks.lookup(probabilities, branch, "probabilities")).each do |item, given|
|
|
198
|
+
odds_key(item, label)
|
|
199
|
+
probability = Checks.nonnegative(given, "#{label}[#{Inputs.shown(item)}]")
|
|
200
|
+
sum = Checks.lookup(shown, item, label, 0.0) + (share * probability)
|
|
201
|
+
Inputs.asked(item, "#{label}: an item's hash") { shown[item] = sum }
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
Checks.finite_answer(shown.values, "the shares and probabilities", shown)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
private
|
|
208
|
+
|
|
209
|
+
# Both Hashes, every branch of `shares` with a Hash of probabilities, and no branch in `probabilities` that
|
|
210
|
+
# `shares` does not name, each branch looked up by the bandit's one lookup in a caller's Hash (Checks.lookup: a
|
|
211
|
+
# default the Hash holds is never read, and the branch's hash and eql? are asked by Inputs.asked).
|
|
212
|
+
def odds_refuse_unpaired(shares, probabilities)
|
|
213
|
+
raise ArgumentError, "shares must be a Hash of branch => share (got #{Inputs.class_name(shares)})" unless Hash === shares
|
|
214
|
+
unless Hash === probabilities
|
|
215
|
+
raise ArgumentError, "probabilities must be a Hash of branch => { item => probability } " \
|
|
216
|
+
"(got #{Inputs.class_name(probabilities)})"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
branches = Inputs.pairs(shares).map(&:first)
|
|
220
|
+
named = Inputs.pairs(probabilities).map(&:first)
|
|
221
|
+
branches.each { |branch| odds_key(branch, "shares") }
|
|
222
|
+
named.each { |branch| odds_key(branch, "probabilities") }
|
|
223
|
+
branches.each do |branch|
|
|
224
|
+
found = Checks.lookup(probabilities, branch, "probabilities")
|
|
225
|
+
next if Hash === found
|
|
226
|
+
|
|
227
|
+
raise ArgumentError, "probabilities must name a Hash of item => probability for the branch " \
|
|
228
|
+
"#{Inputs.shown(branch)} (got #{Inputs.shown(found)})"
|
|
229
|
+
end
|
|
230
|
+
stranger = named.find { |branch| !Checks.holds?(shares, branch, "shares") }
|
|
231
|
+
raise ArgumentError, "probabilities names the branch #{Inputs.shown(stranger)}, which has no share" unless stranger.nil?
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# A branch or an item as given, when a Hash can look it up (the library's one test of a name, Inputs.name?:
|
|
235
|
+
# it answers hash, with an Integer, eql? and inspect, a hash that raises refused there); else an ArgumentError
|
|
236
|
+
# naming `label`.
|
|
237
|
+
def odds_key(key, label)
|
|
238
|
+
return key if Inputs.name?(key, label)
|
|
239
|
+
|
|
240
|
+
raise ArgumentError, "#{label}: #{Inputs.shown(key)} cannot be looked up (a key must answer hash, with an " \
|
|
241
|
+
"Integer, eql? and inspect)"
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# The odds of the pick under its rule (above): { heads:, leaders:, method:, standard_error: }, exact up to
|
|
246
|
+
# EXACT_PICK_MAX contenders, the stated Monte Carlo of P(largest) above.
|
|
247
|
+
def pick_probabilities(draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0)
|
|
248
|
+
Checks.integer(draws, "draws", minimum: 1)
|
|
249
|
+
Checks.integer(seed, "seed")
|
|
250
|
+
rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: {},
|
|
251
|
+
top: nil)
|
|
252
|
+
candidates = contenders(rule[:may_lead])
|
|
253
|
+
exact = candidates.size <= EXACT_PICK_MAX
|
|
254
|
+
refuse_below_smallest_parameter(computing: exact ? ODDS_PICK_EXACT : ODDS_PICK_COUNTED)
|
|
255
|
+
return { heads: {}, leaders: { nil => 1.0 }, method: :exact, standard_error: 0.0 } if candidates.empty?
|
|
256
|
+
|
|
257
|
+
shares = odds_shares(candidates, rule)
|
|
258
|
+
drawn = shares.last
|
|
259
|
+
counted = !exact && drawn.positive?
|
|
260
|
+
largest = if counted
|
|
261
|
+
order_largest_counted(candidates, draws, seed)
|
|
262
|
+
elsif drawn.positive?
|
|
263
|
+
largest_draw(candidates).first
|
|
264
|
+
else
|
|
265
|
+
Array.new(candidates.size, 0.0)
|
|
266
|
+
end
|
|
267
|
+
heads = odds_heads(candidates, rule, shares, largest)
|
|
268
|
+
leaders = odds_leaders(heads, rule[:min_views])
|
|
269
|
+
error = counted ? odds_error(candidates, rule, drawn, largest, draws) : 0.0
|
|
270
|
+
Checks.finite_answer(heads.values + leaders.values << error, "the parameters and the rule",
|
|
271
|
+
{ heads: heads, leaders: leaders, method: counted ? :monte_carlo : :exact,
|
|
272
|
+
standard_error: error })
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# The probability that the rule's whole order begins with `order` (above): { probability:, method:,
|
|
276
|
+
# standard_error: }, exact without items barred from leading up to EXACT_ORDER_MAX free items, else the stated
|
|
277
|
+
# Monte Carlo.
|
|
278
|
+
def order_probability(order, draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0,
|
|
279
|
+
pinned: {}, top: nil)
|
|
280
|
+
Checks.integer(draws, "draws", minimum: 1)
|
|
281
|
+
Checks.integer(seed, "seed")
|
|
282
|
+
rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: pinned,
|
|
283
|
+
top: top)
|
|
284
|
+
named = order_named(order, "order")
|
|
285
|
+
free = rank_free(rule)
|
|
286
|
+
exact = order_exact?(rule, free)
|
|
287
|
+
refuse_below_smallest_parameter(free, computing: exact ? ORDER_EXACT : ORDER_COUNTED)
|
|
288
|
+
if exact
|
|
289
|
+
sequence = order_sequence(named, rule[:pinned])
|
|
290
|
+
probability = nil.equal?(sequence) ? 0.0 : order_exact(sequence, free, rule)
|
|
291
|
+
answer = { probability: probability, method: :exact, standard_error: 0.0 }
|
|
292
|
+
else
|
|
293
|
+
probability, error = order_counted(named, rule, draws, seed)
|
|
294
|
+
answer = { probability: probability, method: :monte_carlo, standard_error: error }
|
|
295
|
+
end
|
|
296
|
+
Checks.finite_answer(answer.values_at(:probability, :standard_error), "the parameters and the rule", answer)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# P(item at each slot) under the rule (above): { probabilities: { item => [Float, one a slot] }, method:,
|
|
300
|
+
# standard_error: }, every item in item order, exact as the orders are, else the stated Monte Carlo. `top` only
|
|
301
|
+
# cuts what `rank` answers, so the table is the whole order's.
|
|
302
|
+
def positions(draws: 20_000, seed: 0, shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0, pinned: {},
|
|
303
|
+
top: nil)
|
|
304
|
+
Checks.integer(draws, "draws", minimum: 1)
|
|
305
|
+
Checks.integer(seed, "seed")
|
|
306
|
+
rule = rank_rule(shuffle: shuffle, explore: explore, may_lead: may_lead, min_views: min_views, pinned: pinned,
|
|
307
|
+
top: top)
|
|
308
|
+
free = rank_free(rule)
|
|
309
|
+
exact = order_exact?(rule, free)
|
|
310
|
+
refuse_below_smallest_parameter(free, computing: exact ? ORDER_EXACT : ORDER_COUNTED)
|
|
311
|
+
if exact
|
|
312
|
+
table = order_slotted(order_ranks(free, rule), free, rule[:pinned])
|
|
313
|
+
error = 0.0
|
|
314
|
+
method = :exact
|
|
315
|
+
else
|
|
316
|
+
table, error = order_counted_positions(rule, draws, seed)
|
|
317
|
+
method = :monte_carlo
|
|
318
|
+
end
|
|
319
|
+
Checks.finite_answer(table.values.flatten << error, "the parameters and the rule",
|
|
320
|
+
{ probabilities: table, method: method, standard_error: error })
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# The posterior alone (above): { probability: P(X_s1 > ... > X_sm > every other item), positions: { item =>
|
|
324
|
+
# [P(at each position by draw)] } }, exact, for a set of at most EXACT_ORDER_MAX items.
|
|
325
|
+
def order_summary(order)
|
|
326
|
+
named = order_named(order, "order")
|
|
327
|
+
if @items.size > EXACT_ORDER_MAX
|
|
328
|
+
raise ArgumentError, "order_summary computes the posterior order exactly among at most #{EXACT_ORDER_MAX} " \
|
|
329
|
+
"items (the set holds #{@items.size})"
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
refuse_below_smallest_parameter(computing: ORDER_POSTERIOR)
|
|
333
|
+
rule = rank_rule(shuffle: 0.0, explore: 0.0, may_lead: nil, min_views: 0.0, pinned: {}, top: nil)
|
|
334
|
+
probability = order_exact(named, @items, rule)
|
|
335
|
+
table = order_slotted(order_ranks(@items, rule), @items, rule[:pinned])
|
|
336
|
+
Checks.finite_answer(table.values.flatten << probability, "the parameters",
|
|
337
|
+
{ probability: probability, positions: table })
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
private
|
|
341
|
+
|
|
342
|
+
# [eligible ({ item => true }, in item order), the explored share, the drawn share] for the contenders under the
|
|
343
|
+
# rule: the explored share is explore when an item is eligible, else 0 (it goes back to the draw); the drawn
|
|
344
|
+
# share what is left, never below 0. The pick's heads and the orders' parts read their shares here.
|
|
345
|
+
def odds_shares(candidates, rule)
|
|
346
|
+
eligible = candidates.select { |item| eligible?(item, rule[:min_views]) }.to_h { |item| [ item, true ] }
|
|
347
|
+
explored = eligible.empty? ? 0.0 : rule[:explore]
|
|
348
|
+
[ eligible, explored, [ 1.0 - rule[:shuffle] - explored, 0.0 ].max ]
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# { item => P(head = item) } for every item: the shuffled share spread over the contenders, the explored share
|
|
352
|
+
# over the eligible, the drawn share by `largest` (P(largest draw) of each contender, in their order), held to
|
|
353
|
+
# [0, 1] (`odds_bounded`).
|
|
354
|
+
def odds_heads(candidates, rule, shares, largest)
|
|
355
|
+
eligible, explored, drawn = shares
|
|
356
|
+
heads = @items.to_h { |item| [ item, 0.0 ] }
|
|
357
|
+
candidates.each_with_index do |item, index|
|
|
358
|
+
head = (rule[:shuffle] / candidates.size) + (eligible.key?(item) ? explored / eligible.size : 0.0) +
|
|
359
|
+
(drawn * largest[index])
|
|
360
|
+
heads[item] = odds_bounded(head)
|
|
361
|
+
end
|
|
362
|
+
heads
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# THE clamp of the odds, written once: a probability held to [0, 1], so a sum that rounds below 0 reads 0 and
|
|
366
|
+
# one that rounds above 1 reads 1 (the pick's heads, the orders' chains and parts, the positions' cells).
|
|
367
|
+
def odds_bounded(probability)
|
|
368
|
+
probability.clamp(0.0, 1.0)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# { item => P(leader = item) } for each item with at least min_views views, then nil => the heads of the rest.
|
|
372
|
+
def odds_leaders(heads, min_views)
|
|
373
|
+
leaders = {}
|
|
374
|
+
without = 0.0
|
|
375
|
+
heads.each do |item, probability|
|
|
376
|
+
if eligible?(item, min_views)
|
|
377
|
+
leaders[item] = probability
|
|
378
|
+
else
|
|
379
|
+
without += probability
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
leaders[nil] = without
|
|
383
|
+
leaders
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# The largest standard error of a head or a leader whose P(largest) part is the Monte Carlo's: the drawn share
|
|
387
|
+
# times sqrt(q (1 - q) / draws), q a contender's share of the draws, or for the leader nil the share summed over
|
|
388
|
+
# the contenders below min_views.
|
|
389
|
+
def odds_error(candidates, rule, drawn, largest, draws)
|
|
390
|
+
without = 0.0
|
|
391
|
+
errors = candidates.each_with_index.map do |item, index|
|
|
392
|
+
without += largest[index] unless eligible?(item, rule[:min_views])
|
|
393
|
+
order_binomial_error(largest[index], draws)
|
|
394
|
+
end
|
|
395
|
+
drawn * (errors << order_binomial_error(without, draws)).max
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
# The Monte Carlo of P(largest) among the contenders: over `draws` draws from Random.new(seed), one Sampler draw
|
|
399
|
+
# per contender in item order (the set's one RNG draw, `sampler_draw`, on the scaled pairs the exact pick odds
|
|
400
|
+
# read), the share in which each contender's draw is the largest by the order rule's own choice of a first
|
|
401
|
+
# largest (`rank_first`, ties to the earlier item) on the keys the rule compares.
|
|
402
|
+
def order_largest_counted(candidates, draws, seed)
|
|
403
|
+
sampler = checked_sampler(Random.new(seed))
|
|
404
|
+
keys = candidates.to_h { |item| [ item, 0.0 ] }
|
|
405
|
+
counts = candidates.to_h { |item| [ item, 0 ] }
|
|
406
|
+
draws.times do
|
|
407
|
+
candidates.each { |item| keys[item] = sampler_draw(sampler, item).first }
|
|
408
|
+
counts[rank_first(candidates, keys, :>)] += 1
|
|
409
|
+
end
|
|
410
|
+
candidates.map { |item| counts.fetch(item).fdiv(draws) }
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
# sqrt(p (1 - p) / draws), a share's standard error over `draws` draws.
|
|
414
|
+
def order_binomial_error(share, draws)
|
|
415
|
+
Math.sqrt([ share * (1.0 - share), 0.0 ].max / draws)
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# The items a caller's order names, read by the set's one reading of a list of item names (`listed_items`), as
|
|
419
|
+
# the set's own Strings in the order's order, frozen; an item named twice is an ArgumentError.
|
|
420
|
+
def order_named(order, label)
|
|
421
|
+
_chosen, named = listed_items(order, label, @items, [])
|
|
422
|
+
seen = {}
|
|
423
|
+
named.each do |item|
|
|
424
|
+
raise ArgumentError, "#{label}: #{Inputs.shown(item)} is named twice" if seen.key?(item)
|
|
425
|
+
|
|
426
|
+
seen[item] = true
|
|
427
|
+
end
|
|
428
|
+
named
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
# Whether the orders and positions are exact under the rule: no free item barred from leading, and at most
|
|
432
|
+
# EXACT_ORDER_MAX free items.
|
|
433
|
+
def order_exact?(rule, free)
|
|
434
|
+
free.size <= EXACT_ORDER_MAX && contenders(rule[:may_lead], free).size == free.size
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
# The free items `named` holds, in its order (each free rank in turn, on the free slots of the free-slot rule,
|
|
438
|
+
# `rank_free_slots`), or nil when the order cannot be the rule's: a pinned slot naming another item, or a pinned
|
|
439
|
+
# item on a free slot.
|
|
440
|
+
def order_sequence(named, pinned)
|
|
441
|
+
return named if pinned.empty?
|
|
442
|
+
return nil if pinned.any? { |slot, item| slot < named.size && named[slot] != item }
|
|
443
|
+
|
|
444
|
+
held = pinned.values
|
|
445
|
+
sequence = rank_free_slots(pinned).take_while { |slot| slot < named.size }.map { |slot| named[slot] }
|
|
446
|
+
sequence.any? { |item| held.include?(item) } ? nil : sequence
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
# P(the free ranks begin with `sequence`) under the rule, exact (no item barred from leading): the shuffled,
|
|
450
|
+
# explored and drawn parts by the shares the pick odds read (`odds_shares`).
|
|
451
|
+
def order_exact(sequence, free, rule)
|
|
452
|
+
return 1.0 if sequence.empty?
|
|
453
|
+
|
|
454
|
+
grid = order_grid(free)
|
|
455
|
+
eligible, explored, drawn = odds_shares(free, rule)
|
|
456
|
+
size = free.size
|
|
457
|
+
probability = 0.0
|
|
458
|
+
if rule[:shuffle].positive?
|
|
459
|
+
arrangements = ((size - sequence.size + 1)..size).reduce(1.0) { |product, count| product * count }
|
|
460
|
+
probability += rule[:shuffle] / arrangements
|
|
461
|
+
end
|
|
462
|
+
if explored.positive? && eligible.key?(sequence.first)
|
|
463
|
+
probability += explored / eligible.size * order_chain(sequence.drop(1), free - sequence.first(1), grid)
|
|
464
|
+
end
|
|
465
|
+
probability += drawn * order_chain(sequence, free, grid) if drawn.positive?
|
|
466
|
+
odds_bounded(probability)
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# T(sequence among `among`) = P(X_s1 > ... > X_sm > every other item of `among`), on the grid: 1 for no item; an
|
|
470
|
+
# order's last item, when no other is left, is the rest; a chain of one item is its P(largest) (the set's one
|
|
471
|
+
# P(largest) by kind); longer chains are the Stieltjes sum on both levels of the grid, each level on its own
|
|
472
|
+
# item's points, reading the level below where it is held (`order_read`), extrapolated.
|
|
473
|
+
def order_chain(sequence, among, grid)
|
|
474
|
+
rest = among - sequence
|
|
475
|
+
if rest.empty?
|
|
476
|
+
rest = sequence.last(1)
|
|
477
|
+
sequence = sequence[0...-1]
|
|
478
|
+
end
|
|
479
|
+
return 1.0 if sequence.empty?
|
|
480
|
+
return order_largest(among)[among.index(sequence.first)] if sequence.size == 1
|
|
481
|
+
|
|
482
|
+
fine, coarse = grid.map do |frames|
|
|
483
|
+
deepest = frames.fetch(sequence.last)
|
|
484
|
+
below = order_product(rest.map { |item| deepest.fetch(item) })
|
|
485
|
+
(sequence.size - 1).downto(1) do |rank|
|
|
486
|
+
item = sequence[rank]
|
|
487
|
+
held = order_step(frames.fetch(item).fetch(item), below)
|
|
488
|
+
below = order_read(frames, item, held, sequence[rank - 1])
|
|
489
|
+
end
|
|
490
|
+
head = sequence.first
|
|
491
|
+
order_total(frames.fetch(head).fetch(head), below)
|
|
492
|
+
end
|
|
493
|
+
odds_bounded(order_extrapolated(fine, coarse))
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
# H of `item`, held at the item's own points (`held`), read at the points of `reader`: as it stands where both
|
|
497
|
+
# share one grid (Betas); else at each of the reader's points mapped into the item's own z, where the grid holds
|
|
498
|
+
# the item's distribution function F at the reader's points (Phi(((the reader's mean - the item's mean) + the
|
|
499
|
+
# reader's sd * z) / the item's sd), the means' difference formed first), linearly in F between the two of the
|
|
500
|
+
# item's own points around it (`order_interpolated`).
|
|
501
|
+
def order_read(frames, item, held, reader)
|
|
502
|
+
own = frames.fetch(item)
|
|
503
|
+
at = frames.fetch(reader)
|
|
504
|
+
return held if own.equal?(at)
|
|
505
|
+
|
|
506
|
+
order_interpolated(own.fetch(item), held, at.fetch(item))
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
# `held` (H at the points where F reads `levels`, both in the points' order, F nondecreasing) read at each of
|
|
510
|
+
# `wanted` (F at the points read), linearly in F between the two points around it: the first point whose F is at
|
|
511
|
+
# least the one wanted, and the one before it. Monotone, and exact at a point whose F is one of `levels`: between
|
|
512
|
+
# two points H moves by the increment of F times a mean of what it integrates, so where F does not move H does not
|
|
513
|
+
# either, and any point of a run of equal F reads one H.
|
|
514
|
+
def order_interpolated(levels, held, wanted)
|
|
515
|
+
size = levels.size
|
|
516
|
+
index = 0
|
|
517
|
+
wanted.map do |level|
|
|
518
|
+
index -= 1 while index.positive? && levels[index - 1] >= level
|
|
519
|
+
index += 1 while index < size - 1 && levels[index] < level
|
|
520
|
+
next held[index] if index.zero? || levels[index] <= level
|
|
521
|
+
|
|
522
|
+
low = levels[index - 1]
|
|
523
|
+
held[index - 1] + ((held[index] - held[index - 1]) * ((level - low) / (levels[index] - low)))
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
# The Stieltjes sum extrapolated from the grid (`fine`) and its every other level (`coarse`): fine + (fine -
|
|
528
|
+
# coarse) / 3, which takes out the trapezoid's error that falls as the square of the spacing (Richardson).
|
|
529
|
+
def order_extrapolated(fine, coarse)
|
|
530
|
+
fine + ((fine - coarse) / 3.0)
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
# The pointwise product of the columns (each item's distribution function on the grid).
|
|
534
|
+
def order_product(columns)
|
|
535
|
+
return columns.first if columns.size == 1
|
|
536
|
+
|
|
537
|
+
Array.new(columns.first.size) do |point|
|
|
538
|
+
columns.inject(1.0) { |product, column| product * column[point] }
|
|
539
|
+
end
|
|
540
|
+
end
|
|
541
|
+
|
|
542
|
+
# THE trapezoid's Stieltjes increment, written once (`order_step`, `order_total` and `order_accumulate` sum it):
|
|
543
|
+
# the integrator's increment over an interval times the mean of the integrand at the interval's two ends.
|
|
544
|
+
def order_trapezoid(increment, before, now)
|
|
545
|
+
increment * (before + now) * 0.5
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
# One Stieltjes step: H_k at every grid point, from its item's distribution function `own` and H_(k+1) `below`,
|
|
549
|
+
# 0 at the first point.
|
|
550
|
+
def order_step(own, below)
|
|
551
|
+
size = own.size
|
|
552
|
+
step = Array.new(size, 0.0)
|
|
553
|
+
point = 1
|
|
554
|
+
while point < size
|
|
555
|
+
step[point] = step[point - 1] + order_trapezoid(own[point] - own[point - 1], below[point - 1], below[point])
|
|
556
|
+
point += 1
|
|
557
|
+
end
|
|
558
|
+
step
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# The last Stieltjes step's value at the top of the grid.
|
|
562
|
+
def order_total(own, below)
|
|
563
|
+
order_step(own, below).last
|
|
564
|
+
end
|
|
565
|
+
|
|
566
|
+
# { item => [P(item at free rank l), l = 0 ... n - 1] } for the free items under the rule, exact (no item barred
|
|
567
|
+
# from leading): the shuffled part 1/n at every rank, the explored and drawn parts by rank (above).
|
|
568
|
+
def order_ranks(free, rule)
|
|
569
|
+
return {} if free.empty?
|
|
570
|
+
return { free.first => [ 1.0 ] } if free.size == 1
|
|
571
|
+
|
|
572
|
+
size = free.size
|
|
573
|
+
eligible, explored, drawn = odds_shares(free, rule)
|
|
574
|
+
grid = order_grid(free)
|
|
575
|
+
largest = order_largest(free)
|
|
576
|
+
sums = order_coefficient_sums(free, grid, eligible, explored.positive?)
|
|
577
|
+
free.each_with_index.to_h do |item, index|
|
|
578
|
+
by_draw, by_explore = sums.fetch(item)
|
|
579
|
+
row = Array.new(size) do |rank|
|
|
580
|
+
probability = rule[:shuffle] / size
|
|
581
|
+
probability += drawn * (rank.zero? ? largest[index] : by_draw[rank]) if drawn.positive?
|
|
582
|
+
if explored.positive?
|
|
583
|
+
probability += explored * order_explored(item, rank, free, eligible, by_explore) / eligible.size
|
|
584
|
+
end
|
|
585
|
+
odds_bounded(probability)
|
|
586
|
+
end
|
|
587
|
+
[ item, row ]
|
|
588
|
+
end
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
# |E| times the explored part's P(item at `rank`): 1 at rank 0 for an eligible item; at rank 1 the sum over the
|
|
592
|
+
# eligible h != item of P(item is the largest among the free items less h), the set's one P(largest); deeper, the
|
|
593
|
+
# Stieltjes sum of the coefficient over those h.
|
|
594
|
+
def order_explored(item, rank, free, eligible, by_explore)
|
|
595
|
+
return eligible.key?(item) ? 1.0 : 0.0 if rank.zero?
|
|
596
|
+
return by_explore[rank - 1] unless rank == 1
|
|
597
|
+
|
|
598
|
+
eligible.each_key.sum do |head|
|
|
599
|
+
next 0.0 if head == item
|
|
600
|
+
|
|
601
|
+
among = free - [ head ]
|
|
602
|
+
order_largest(among)[among.index(item)]
|
|
603
|
+
end
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
# { item => [the Stieltjes sums by draw, one a rank, and by explore, one a rank less one] }. At every one of the
|
|
607
|
+
# item's own points, the coefficients of the product over the other free items j of (F_j + z (1 - F_j)) (by draw)
|
|
608
|
+
# and, when `explored`, of the sum over the eligible h of that product without h (by explore); each increment of
|
|
609
|
+
# the item's own F times the mean of the coefficients at its two ends; on both levels of the grid, extrapolated
|
|
610
|
+
# (`order_extrapolated`).
|
|
611
|
+
def order_coefficient_sums(free, grid, eligible, explored)
|
|
612
|
+
free.to_h do |item|
|
|
613
|
+
others = free.reject { |other| other == item }
|
|
614
|
+
flags = others.map { |other| eligible.key?(other) }
|
|
615
|
+
fine, coarse = grid.map do |frames|
|
|
616
|
+
columns = frames.fetch(item)
|
|
617
|
+
order_item_sums(columns.fetch(item), others.map { |other| columns.fetch(other) }, flags, explored)
|
|
618
|
+
end
|
|
619
|
+
sums = fine.zip(coarse).map do |by_fine, by_coarse|
|
|
620
|
+
by_fine.zip(by_coarse).map { |sum, half| order_extrapolated(sum, half) }
|
|
621
|
+
end
|
|
622
|
+
[ item, sums ]
|
|
623
|
+
end
|
|
624
|
+
end
|
|
625
|
+
|
|
626
|
+
# One item's sums (`order_coefficient_sums`): `own` its distribution function on its points, `columns` the other
|
|
627
|
+
# items' there, `flags` whether each other is eligible. The products are built factor by factor in place, highest
|
|
628
|
+
# coefficient first, the explored sum before the product it reads (B' = B (f + c z) + [eligible] A,
|
|
629
|
+
# A' = A (f + c z)).
|
|
630
|
+
def order_item_sums(own, columns, flags, explored)
|
|
631
|
+
size = columns.size + 1
|
|
632
|
+
product = Array.new(size, 0.0)
|
|
633
|
+
product_before = Array.new(size, 0.0)
|
|
634
|
+
without = Array.new(size, 0.0)
|
|
635
|
+
without_before = Array.new(size, 0.0)
|
|
636
|
+
by_draw = Array.new(size, 0.0)
|
|
637
|
+
by_explore = Array.new(size, 0.0)
|
|
638
|
+
own.each_index do |point|
|
|
639
|
+
product.fill(0.0)
|
|
640
|
+
product[0] = 1.0
|
|
641
|
+
without.fill(0.0) if explored
|
|
642
|
+
columns.each_with_index do |column, factor|
|
|
643
|
+
below = column[point]
|
|
644
|
+
above = 1.0 - below
|
|
645
|
+
order_add_factor(without, product, factor, below, above, flags[factor]) if explored
|
|
646
|
+
order_multiply(product, factor, below, above)
|
|
647
|
+
end
|
|
648
|
+
if point.positive?
|
|
649
|
+
increment = own[point] - own[point - 1]
|
|
650
|
+
unless increment.zero?
|
|
651
|
+
order_accumulate(by_draw, increment, product_before, product)
|
|
652
|
+
order_accumulate(by_explore, increment, without_before, without) if explored
|
|
653
|
+
end
|
|
654
|
+
end
|
|
655
|
+
product, product_before = product_before, product
|
|
656
|
+
without, without_before = without_before, without
|
|
657
|
+
end
|
|
658
|
+
[ by_draw, by_explore ]
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
# `coefficients` (degree `factor`, held in 0..factor) times (below + above z), in place.
|
|
662
|
+
def order_multiply(coefficients, factor, below, above)
|
|
663
|
+
coefficients[factor + 1] = above * coefficients[factor]
|
|
664
|
+
degree = factor
|
|
665
|
+
while degree.positive?
|
|
666
|
+
coefficients[degree] = (below * coefficients[degree]) + (above * coefficients[degree - 1])
|
|
667
|
+
degree -= 1
|
|
668
|
+
end
|
|
669
|
+
coefficients[0] *= below
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
# The explored sum B (degree factor - 1) times (below + above z), plus the product A as it stands when the
|
|
673
|
+
# factor's item is eligible, in place.
|
|
674
|
+
def order_add_factor(sum, product, factor, below, above, eligible)
|
|
675
|
+
degree = factor
|
|
676
|
+
while degree >= 0
|
|
677
|
+
value = below * sum[degree]
|
|
678
|
+
value += above * sum[degree - 1] if degree.positive?
|
|
679
|
+
value += product[degree] if eligible
|
|
680
|
+
sum[degree] = value
|
|
681
|
+
degree -= 1
|
|
682
|
+
end
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
# Each coefficient's sum plus its trapezoid increment (`order_trapezoid`) over the interval.
|
|
686
|
+
def order_accumulate(sums, increment, before, now)
|
|
687
|
+
index = 0
|
|
688
|
+
size = sums.size
|
|
689
|
+
while index < size
|
|
690
|
+
sums[index] += order_trapezoid(increment, before[index], now[index])
|
|
691
|
+
index += 1
|
|
692
|
+
end
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
# The rank table placed on the slots: each free rank on the next free slot (the free-slot rule,
|
|
696
|
+
# `rank_free_slots`), a pinned item 1 at its own slot, every item in item order, a row a slot.
|
|
697
|
+
def order_slotted(ranks, free, pinned)
|
|
698
|
+
slots = rank_free_slots(pinned)
|
|
699
|
+
table = @items.to_h { |item| [ item, Array.new(@items.size, 0.0) ] }
|
|
700
|
+
pinned.each { |slot, item| table.fetch(item)[slot] = 1.0 }
|
|
701
|
+
free.each { |item| ranks.fetch(item).each_with_index { |probability, rank| table.fetch(item)[slots[rank]] = probability } }
|
|
702
|
+
table
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
# The grid of the free items' order odds, built once a set and kept, on two levels: [fine, coarse], each { item
|
|
706
|
+
# => { free item => its distribution function F at each of the item's points } }, the coarse level a part of the
|
|
707
|
+
# fine one's points: the infinities, the quantiles at every other level (2/ORDER_LEVELS, 4/ORDER_LEVELS, ...)
|
|
708
|
+
# and, for Normals, the levels 0 and 1 and the points beside each item's carried points. Betas share one grid
|
|
709
|
+
# (`order_beta_grid`), so every item names one Hash; Normals have each item's own (`order_normal_grid`). F is 0
|
|
710
|
+
# at -infinity and 1 at +infinity. The free items' P(largest) is read first, the one guard of what the integrals
|
|
711
|
+
# can read.
|
|
712
|
+
def order_grid(free)
|
|
713
|
+
order_memo(:grid, free) do
|
|
714
|
+
order_largest(free)
|
|
715
|
+
@kind == :beta ? order_beta_grid(free) : order_normal_grid(free)
|
|
716
|
+
end
|
|
717
|
+
end
|
|
718
|
+
|
|
719
|
+
# The Betas' one grid, in the position the rule compares their draws by (BetaArgmax.position's log scale,
|
|
720
|
+
# ranking.rb): -infinity, +infinity and every free item's position at the ORDER_LEVELS - 1 levels inside (0, 1)
|
|
721
|
+
# on its scaled pair (where a draw at that uniform compares), each item's F read there
|
|
722
|
+
# (`order_beta_distribution`).
|
|
723
|
+
def order_beta_grid(free)
|
|
724
|
+
tagged = [ [ -Float::INFINITY, true ], [ Float::INFINITY, true ] ]
|
|
725
|
+
free.each do |item|
|
|
726
|
+
alpha, beta = @parameters.fetch(item)
|
|
727
|
+
(1...ORDER_LEVELS).each do |level|
|
|
728
|
+
tagged << [ BetaArgmax.position(level.fdiv(ORDER_LEVELS), alpha, beta), level.even? ]
|
|
729
|
+
end
|
|
730
|
+
end
|
|
731
|
+
points, coarse = order_merged(tagged)
|
|
732
|
+
fine = free.to_h do |item|
|
|
733
|
+
alpha, beta = @parameters.fetch(item)
|
|
734
|
+
[ item, points.map { |position| order_beta_distribution(position, alpha, beta) }.freeze ]
|
|
735
|
+
end.freeze
|
|
736
|
+
thinned = fine.transform_values { |column| column.values_at(*coarse).freeze }.freeze
|
|
737
|
+
[ free.to_h { |item| [ item, fine ] }.freeze, free.to_h { |item| [ item, thinned ] }.freeze ].freeze
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
# Each Normal's own points, in its own standardised variable z = (x - mean) / sd on its scaled pair (as
|
|
741
|
+
# NormalArgmax.probabilities integrates each member): -infinity, +infinity, and every free item's quantile at
|
|
742
|
+
# each level 0, 1/ORDER_LEVELS, ..., 1 (the standard quantile at the level read as a quantile draw's uniform is,
|
|
743
|
+
# Draw.open_uniform, so the levels 0 and 1 read the open grid's ends, 2**-53 within them) carried into z, ((the
|
|
744
|
+
# other's mean - the item's mean) + the other's sd * the standard quantile) / the item's sd, the means'
|
|
745
|
+
# difference formed first, where it is finite; beside each item's carried points the second Float below the
|
|
746
|
+
# lowest and above the highest, so an item whose points round together in z rises wholly between two points. An
|
|
747
|
+
# item's own ends keep its mass between -infinity and its first quantile at 2**-53, where the other items' F
|
|
748
|
+
# reads 0 by the formula whatever their scale. Every free item's F is read at those points
|
|
749
|
+
# (`order_normal_column`).
|
|
750
|
+
def order_normal_grid(free)
|
|
751
|
+
scores = (0..ORDER_LEVELS).map do |level|
|
|
752
|
+
Significance.normal_quantile(Draw.open_uniform(level.fdiv(ORDER_LEVELS)))
|
|
753
|
+
end
|
|
754
|
+
own = free.map do |item|
|
|
755
|
+
mean, deviation = @parameters.fetch(item)
|
|
756
|
+
tagged = [ [ -Float::INFINITY, true ], [ Float::INFINITY, true ] ]
|
|
757
|
+
free.each { |other| order_carried(tagged, scores, mean, deviation, @parameters.fetch(other)) }
|
|
758
|
+
points, coarse = order_merged(tagged)
|
|
759
|
+
columns = free.to_h do |other|
|
|
760
|
+
[ other, order_normal_column(points, mean, deviation, @parameters.fetch(other)).freeze ]
|
|
761
|
+
end
|
|
762
|
+
[ columns.freeze, columns.transform_values { |column| column.values_at(*coarse).freeze }.freeze ]
|
|
763
|
+
end
|
|
764
|
+
[ free.zip(own.map(&:first)).to_h.freeze, free.zip(own.map(&:last)).to_h.freeze ].freeze
|
|
765
|
+
end
|
|
766
|
+
|
|
767
|
+
# The other item's quantiles carried into the item's z (`order_normal_grid`) by the one map between two Normals'
|
|
768
|
+
# standardised variables (Standardised.carried, the means' difference formed first), each tagged coarse at every
|
|
769
|
+
# other level (the ends included), and the two points beside them, tagged coarse, added to `tagged`.
|
|
770
|
+
def order_carried(tagged, scores, mean, deviation, (other_mean, other_deviation))
|
|
771
|
+
lowest = Float::INFINITY
|
|
772
|
+
highest = -Float::INFINITY
|
|
773
|
+
scores.each_with_index do |score, index|
|
|
774
|
+
z = Standardised.carried(other_mean, other_deviation, score, mean, deviation)
|
|
775
|
+
next unless z.finite?
|
|
776
|
+
|
|
777
|
+
tagged << [ z, index.even? ]
|
|
778
|
+
lowest = z if z < lowest
|
|
779
|
+
highest = z if z > highest
|
|
780
|
+
end
|
|
781
|
+
return unless lowest.finite?
|
|
782
|
+
|
|
783
|
+
tagged << [ lowest.prev_float.prev_float, true ] << [ highest.next_float.next_float, true ]
|
|
784
|
+
end
|
|
785
|
+
|
|
786
|
+
# The other item's F at each of the item's points z: Phi(((the item's mean - the other's mean) + the item's sd *
|
|
787
|
+
# z) / the other's sd), the item's z carried into the other's by the one map (Standardised.carried, the means'
|
|
788
|
+
# difference formed first), so a finite numerator over a standard deviation reads 0 or 1 where the quotient
|
|
789
|
+
# leaves a Float, never a NaN (0 at -infinity, 1 at +infinity).
|
|
790
|
+
def order_normal_column(points, mean, deviation, (other_mean, other_deviation))
|
|
791
|
+
points.map do |z|
|
|
792
|
+
Significance.normal_cdf(Standardised.carried(mean, deviation, z, other_mean, other_deviation))
|
|
793
|
+
end
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
# [the points, sorted, each once; the indices of the coarse ones] from [point, coarse?] pairs: a point given more
|
|
797
|
+
# than once is coarse when any of its givings is.
|
|
798
|
+
def order_merged(tagged)
|
|
799
|
+
points = []
|
|
800
|
+
coarse = []
|
|
801
|
+
tagged.sort_by(&:first).each do |point, half|
|
|
802
|
+
if points.empty? || points.last != point
|
|
803
|
+
points << point
|
|
804
|
+
coarse << half
|
|
805
|
+
elsif half
|
|
806
|
+
coarse[-1] = true
|
|
807
|
+
end
|
|
808
|
+
end
|
|
809
|
+
[ points.freeze, coarse.each_index.select { |index| coarse[index] }.freeze ]
|
|
810
|
+
end
|
|
811
|
+
|
|
812
|
+
# Beta(alpha, beta)'s distribution function at the position p (x = exp(p + JOINT) below 0, else 1 -
|
|
813
|
+
# exp(JOINT - p): the position's two halves, `rank_beta_half`, as the quantile draws read them), read in each
|
|
814
|
+
# half from its own end, as BetaArgmax reads its two halves: below 0 the item's own at log x = p + JOINT, from 0
|
|
815
|
+
# up 1 less the reflected item's (beta, alpha) at log (1 - x) = JOINT - p (`order_beta_end`). So no mass near
|
|
816
|
+
# either end rounds into the end itself.
|
|
817
|
+
def order_beta_distribution(position, alpha, beta)
|
|
818
|
+
lower, log_end = rank_beta_half(position)
|
|
819
|
+
return order_beta_end(log_end, alpha, beta) if lower
|
|
820
|
+
|
|
821
|
+
1.0 - order_beta_end(log_end, beta, alpha)
|
|
822
|
+
end
|
|
823
|
+
|
|
824
|
+
# I_w(p, q) at log w = v (v at most JOINT, -infinity included): BetaArgmax.distribution at w = exp(v) where w is
|
|
825
|
+
# at least ORDER_FLOOR, the function BetaArgmax integrates and positions on. Below it, where a Float holds fewer
|
|
826
|
+
# digits of w and then none, the distribution function is its leading term w**p / (p * B(p, q)) to a Float's
|
|
827
|
+
# precision (the next term is about q * w of it, below 1e-301 within the scaled pairs' total), a power of w:
|
|
828
|
+
# BetaArgmax.distribution at ORDER_FLOOR times exp(p * (v - log ORDER_FLOOR)), a factor at most 1.
|
|
829
|
+
def order_beta_end(v, p, q)
|
|
830
|
+
return BetaArgmax.distribution(Math.exp(v), p, q) if v >= ORDER_LOG_FLOOR
|
|
831
|
+
|
|
832
|
+
BetaArgmax.distribution(ORDER_FLOOR, p, q) * Math.exp(p * (v - ORDER_LOG_FLOOR))
|
|
833
|
+
end
|
|
834
|
+
|
|
835
|
+
# P(largest) of each item of `among` (in its order), the set's one P(largest) by kind, kept once read.
|
|
836
|
+
def order_largest(among)
|
|
837
|
+
order_memo(:largest, among) { largest_draw(among).first.freeze }
|
|
838
|
+
end
|
|
839
|
+
|
|
840
|
+
# The value kept on this set under `what` for the items `among` (a frozen copy of the list), computed by the
|
|
841
|
+
# block once (a set the caller froze keeps nothing and computes each time).
|
|
842
|
+
def order_memo(what, among)
|
|
843
|
+
return yield if frozen?
|
|
844
|
+
|
|
845
|
+
kept = (@order_kept ||= {})
|
|
846
|
+
key = [ what, among.frozen? ? among : among.dup.freeze ].freeze
|
|
847
|
+
kept.fetch(key) { kept[key] = yield }
|
|
848
|
+
end
|
|
849
|
+
|
|
850
|
+
# The rule's visits from the RNG source: `draws` whole orders (the order rule's own ordering, as `rank(rng:)`
|
|
851
|
+
# orders, pinned items at their slots, its Sampler draws on the scaled pairs, `parameters`, the pairs the exact
|
|
852
|
+
# order odds read, compared as the rule compares them) drawn on Random.new(seed), each yielded.
|
|
853
|
+
def order_visits(rule, draws, seed)
|
|
854
|
+
rng = Random.new(seed)
|
|
855
|
+
draws.times { yield rank_ordered(rule, nil, nil, rng).first }
|
|
856
|
+
end
|
|
857
|
+
|
|
858
|
+
# [the share of the rule's visits whose whole order begins with `named`, its standard error].
|
|
859
|
+
def order_counted(named, rule, draws, seed)
|
|
860
|
+
length = named.size
|
|
861
|
+
hits = 0
|
|
862
|
+
order_visits(rule, draws, seed) { |whole| hits += 1 if whole.first(length) == named }
|
|
863
|
+
share = hits.fdiv(draws)
|
|
864
|
+
[ share, order_binomial_error(share, draws) ]
|
|
865
|
+
end
|
|
866
|
+
|
|
867
|
+
# [{ item => [the share of visits with the item at each slot] }, the largest standard error of those shares].
|
|
868
|
+
def order_counted_positions(rule, draws, seed)
|
|
869
|
+
size = @items.size
|
|
870
|
+
counts = @items.to_h { |item| [ item, Array.new(size, 0) ] }
|
|
871
|
+
order_visits(rule, draws, seed) do |whole|
|
|
872
|
+
whole.each_with_index { |item, slot| counts.fetch(item)[slot] += 1 }
|
|
873
|
+
end
|
|
874
|
+
error = 0.0
|
|
875
|
+
table = counts.transform_values do |row|
|
|
876
|
+
row.map do |count|
|
|
877
|
+
share = count.fdiv(draws)
|
|
878
|
+
error = [ error, order_binomial_error(share, draws) ].max
|
|
879
|
+
share
|
|
880
|
+
end
|
|
881
|
+
end
|
|
882
|
+
[ table, error ]
|
|
883
|
+
end
|
|
884
|
+
end
|
|
885
|
+
|
|
886
|
+
# What a Choice or an Order carries from the set that made it (ranking.rb sets it before the freeze): [the set,
|
|
887
|
+
# the rule, the cache]; an ArgumentError for one made by hand, which carries none.
|
|
888
|
+
module OrderCarried
|
|
889
|
+
module_function
|
|
890
|
+
|
|
891
|
+
def read(answer)
|
|
892
|
+
posteriors = answer.instance_variable_get(:@posteriors)
|
|
893
|
+
rule = answer.instance_variable_get(:@rule)
|
|
894
|
+
cache = answer.instance_variable_get(:@cache)
|
|
895
|
+
return [ posteriors, rule, cache ] if Posteriors === posteriors && Hash === rule && Hash === cache
|
|
896
|
+
|
|
897
|
+
raise ArgumentError, "a #{Inputs.class_name(answer)} answers its probability only when a posterior set made " \
|
|
898
|
+
"it (it carries the set, the rule and the cache)"
|
|
899
|
+
end
|
|
900
|
+
end
|
|
901
|
+
private_constant :OrderCarried
|
|
902
|
+
|
|
903
|
+
# A pick's odds under the rule that made it (probabilities.rb).
|
|
904
|
+
class Choice
|
|
905
|
+
# P(head = this head) under the rule the pick was made by: its set's `pick_probabilities` of the head (at the
|
|
906
|
+
# default draws and seed where they are a Monte Carlo), 1.0 for no head (an empty set). Computed once and kept
|
|
907
|
+
# in the cache it carries.
|
|
908
|
+
def probability
|
|
909
|
+
posteriors, rule, cache = OrderCarried.read(self)
|
|
910
|
+
cache.fetch(:probability) do
|
|
911
|
+
cache[:probability] = head.nil? ? 1.0 : posteriors.pick_probabilities(**rule)[:heads].fetch(head)
|
|
912
|
+
end
|
|
913
|
+
end
|
|
914
|
+
end
|
|
915
|
+
|
|
916
|
+
# An order's odds under the rule that made it (probabilities.rb).
|
|
917
|
+
class Order
|
|
918
|
+
# P(the rule's order begins with these items) under the rule the order was made by: its set's
|
|
919
|
+
# `order_probability` of its items (at the default draws and seed where they are a Monte Carlo). Computed once
|
|
920
|
+
# and kept in the cache it carries.
|
|
921
|
+
def probability
|
|
922
|
+
posteriors, rule, cache = OrderCarried.read(self)
|
|
923
|
+
cache.fetch(:probability) do
|
|
924
|
+
cache[:probability] = posteriors.order_probability(items, **rule)[:probability]
|
|
925
|
+
end
|
|
926
|
+
end
|
|
927
|
+
end
|
|
928
|
+
end
|
|
929
|
+
end
|