bandit_machines 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +95 -0
- data/Cargo.lock +321 -0
- data/Cargo.toml +29 -0
- data/LICENSE.txt +21 -0
- data/README.md +367 -0
- data/ext/bandit_machines_native/core/CHANGELOG.md +70 -0
- data/ext/bandit_machines_native/core/Cargo.toml +21 -0
- data/ext/bandit_machines_native/core/README.md +152 -0
- data/ext/bandit_machines_native/core/src/bandit.rs +576 -0
- data/ext/bandit_machines_native/core/src/lib.rs +286 -0
- data/ext/bandit_machines_native/core/src/math.rs +66 -0
- data/ext/bandit_machines_native/core/src/pick.rs +71 -0
- data/ext/bandit_machines_native/extconf.rb +2 -0
- data/ext/bandit_machines_native/ffi/Cargo.toml +21 -0
- data/ext/bandit_machines_native/ffi/extconf.rb +14 -0
- data/ext/bandit_machines_native/ffi/src/convert.rs +238 -0
- data/ext/bandit_machines_native/ffi/src/init.rs +128 -0
- data/ext/bandit_machines_native/ffi/src/lib.rs +246 -0
- data/lib/bandit_machines/version.rb +5 -0
- data/lib/bandit_machines.rb +74 -0
- data/sig/bandit_machines.rbs +157 -0
- metadata +87 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
//! The bounded bandit: arms, outstanding tickets and the two policies.
|
|
2
|
+
|
|
3
|
+
use core::cmp::Ordering;
|
|
4
|
+
|
|
5
|
+
use periscope_machines::{Issuer, scope_of};
|
|
6
|
+
|
|
7
|
+
use crate::math::{bonus_q56, mean_q56};
|
|
8
|
+
use crate::pick::{Candidate, best, draw_below};
|
|
9
|
+
use crate::{
|
|
10
|
+
ArmStats, Closing, Config, Draws, Opening, Ratio, Reason, Refusal, Selection, Ticket, TieBreak,
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
/// One arm's state is exactly the snapshot [`Bandit::stats`] hands out.
|
|
14
|
+
impl ArmStats {
|
|
15
|
+
/// A closed arm with nothing observed, generation zero.
|
|
16
|
+
const NEW: Self = Self {
|
|
17
|
+
open: false,
|
|
18
|
+
generation: 0,
|
|
19
|
+
rewarded: 0,
|
|
20
|
+
reward_sum: 0,
|
|
21
|
+
pending: 0,
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/// The arm after a reset: eligibility kept, everything else new, one
|
|
25
|
+
/// generation on.
|
|
26
|
+
fn forgotten(self) -> Result<Self, Refusal> {
|
|
27
|
+
Ok(Self {
|
|
28
|
+
open: self.open,
|
|
29
|
+
generation: self.generation.checked_add(1).ok_or(Refusal::Exhausted)?,
|
|
30
|
+
..Self::NEW
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
36
|
+
struct Pending {
|
|
37
|
+
arm: usize,
|
|
38
|
+
serial: u64,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/// An exact mean `sum / count`, ordered by cross-multiplication.
|
|
42
|
+
#[derive(Debug, Clone, Copy)]
|
|
43
|
+
struct Mean {
|
|
44
|
+
sum: u64,
|
|
45
|
+
count: u64,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
impl Ord for Mean {
|
|
49
|
+
fn cmp(&self, other: &Self) -> Ordering {
|
|
50
|
+
let left = u128::from(self.sum) * u128::from(other.count);
|
|
51
|
+
let right = u128::from(other.sum) * u128::from(self.count);
|
|
52
|
+
left.cmp(&right)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
impl PartialOrd for Mean {
|
|
57
|
+
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
|
58
|
+
Some(self.cmp(other))
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
impl PartialEq for Mean {
|
|
63
|
+
fn eq(&self, other: &Self) -> bool {
|
|
64
|
+
self.cmp(other) == Ordering::Equal
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl Eq for Mean {}
|
|
69
|
+
|
|
70
|
+
/// `ARMS` arms and at most `TICKETS` selections awaiting their reward.
|
|
71
|
+
///
|
|
72
|
+
/// See the crate documentation for the cold-start, tie, delayed-reward and
|
|
73
|
+
/// nonstationarity contracts.
|
|
74
|
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
75
|
+
pub struct Bandit<const ARMS: usize, const TICKETS: usize> {
|
|
76
|
+
pub(crate) config: Config,
|
|
77
|
+
arms: [ArmStats; ARMS],
|
|
78
|
+
tickets: [Option<Pending>; TICKETS],
|
|
79
|
+
/// Issues ticket serials; never reissues one.
|
|
80
|
+
issuer: Issuer,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
impl<const ARMS: usize, const TICKETS: usize> Bandit<ARMS, TICKETS> {
|
|
84
|
+
/// Compiled number of arms.
|
|
85
|
+
pub const ARMS: usize = ARMS;
|
|
86
|
+
|
|
87
|
+
/// Compiled number of outstanding tickets.
|
|
88
|
+
pub const TICKETS: usize = TICKETS;
|
|
89
|
+
|
|
90
|
+
/// A bandit with every arm closed and nothing observed, in scope zero.
|
|
91
|
+
#[must_use]
|
|
92
|
+
pub const fn new(config: Config) -> Self {
|
|
93
|
+
Self::with_scope(config, 0)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// A bandit with every arm closed and nothing observed, whose tickets
|
|
97
|
+
/// carry `scope`.
|
|
98
|
+
///
|
|
99
|
+
/// The scope is the bandit's identity, not policy, so it is not part of
|
|
100
|
+
/// [`Config`]: replicas share a configuration and must not share a scope.
|
|
101
|
+
/// Tickets are only meaningful to the bandit that issued them, and two
|
|
102
|
+
/// bandits in the same scope issue the same tickets.
|
|
103
|
+
#[must_use]
|
|
104
|
+
pub const fn with_scope(config: Config, scope: u16) -> Self {
|
|
105
|
+
Self::build(config, Issuer::new(scope))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/// A bandit that continues `scope` after a restart: `issued` is what
|
|
109
|
+
/// [`issued`](Self::issued) answered when the caller saved it.
|
|
110
|
+
///
|
|
111
|
+
/// Only the serial sequence is restored. Every arm is closed with nothing
|
|
112
|
+
/// observed and generation zero, and no ticket is outstanding: a ticket
|
|
113
|
+
/// issued before the restart is refused ([`Refusal::UnknownTicket`], or
|
|
114
|
+
/// [`Refusal::StaleTicket`] when its arm's generation differs), and
|
|
115
|
+
/// reopening arms and restoring what they learned is the caller's job.
|
|
116
|
+
/// The next ticket carries sequence `issued + 1`, so no serial is
|
|
117
|
+
/// reissued as long as the saved count is not lower than what the bandit
|
|
118
|
+
/// had really issued; save it before handing a ticket out.
|
|
119
|
+
///
|
|
120
|
+
/// `None` when `issued` is beyond `periscope_machines::SEQUENCE_MAX`,
|
|
121
|
+
/// which no bandit reaches.
|
|
122
|
+
#[must_use]
|
|
123
|
+
pub const fn resumed(config: Config, scope: u16, issued: u64) -> Option<Self> {
|
|
124
|
+
match Issuer::resumed(scope, issued) {
|
|
125
|
+
Some(sequence) => Some(Self::build(config, sequence)),
|
|
126
|
+
None => None,
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const fn build(config: Config, issuer: Issuer) -> Self {
|
|
131
|
+
Self {
|
|
132
|
+
config,
|
|
133
|
+
arms: [ArmStats::NEW; ARMS],
|
|
134
|
+
tickets: [None; TICKETS],
|
|
135
|
+
issuer,
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/// The scope carried by every ticket this bandit issues.
|
|
140
|
+
#[must_use]
|
|
141
|
+
pub const fn scope(&self) -> u16 {
|
|
142
|
+
self.issuer.scope()
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/// Serials issued in this scope so far, outstanding or not: the count to
|
|
146
|
+
/// save for [`resumed`](Self::resumed).
|
|
147
|
+
#[must_use]
|
|
148
|
+
pub const fn issued(&self) -> u64 {
|
|
149
|
+
self.issuer.issued()
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/// The configuration this bandit was built with.
|
|
153
|
+
#[must_use]
|
|
154
|
+
pub const fn config(&self) -> Config {
|
|
155
|
+
self.config
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/// Tickets issued and not yet rewarded, abandoned, revoked or voided.
|
|
159
|
+
#[must_use]
|
|
160
|
+
pub const fn outstanding(&self) -> usize {
|
|
161
|
+
// Counted, not kept: a separate counter would be one more invariant.
|
|
162
|
+
let mut count = 0;
|
|
163
|
+
let mut rest = self.tickets.as_slice();
|
|
164
|
+
while let [slot, tail @ ..] = rest {
|
|
165
|
+
if slot.is_some() {
|
|
166
|
+
count += 1;
|
|
167
|
+
}
|
|
168
|
+
rest = tail;
|
|
169
|
+
}
|
|
170
|
+
count
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/// Serial the next ticket will carry; every issued serial is below it.
|
|
174
|
+
/// `None` once every serial of the scope has been issued.
|
|
175
|
+
#[must_use]
|
|
176
|
+
pub const fn next_serial(&self) -> Option<u64> {
|
|
177
|
+
match self.issuer.peek() {
|
|
178
|
+
Ok(serial) => Some(serial),
|
|
179
|
+
Err(periscope_machines::Refusal::Exhausted) => None,
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// Snapshot of one arm, or `None` past the compiled capacity.
|
|
184
|
+
#[must_use]
|
|
185
|
+
pub fn stats(&self, arm: usize) -> Option<ArmStats> {
|
|
186
|
+
self.arms.get(arm).copied()
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/// Make an arm eligible for selection.
|
|
190
|
+
///
|
|
191
|
+
/// [`Opening::AlreadyOpen`] is an idempotent repeat: it changes nothing.
|
|
192
|
+
///
|
|
193
|
+
/// # Errors
|
|
194
|
+
/// [`Refusal::UnknownArm`] past the compiled capacity.
|
|
195
|
+
pub fn open(&mut self, arm: usize) -> Result<Opening, Refusal> {
|
|
196
|
+
let arm = self.arms.get_mut(arm).ok_or(Refusal::UnknownArm)?;
|
|
197
|
+
if arm.open {
|
|
198
|
+
return Ok(Opening::AlreadyOpen);
|
|
199
|
+
}
|
|
200
|
+
arm.open = true;
|
|
201
|
+
Ok(Opening::Opened)
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/// Make an arm ineligible.
|
|
205
|
+
///
|
|
206
|
+
/// Closing is eligibility only: statistics stay, and outstanding tickets
|
|
207
|
+
/// for the arm are still rewardable. [`Closing::AlreadyClosed`] is an
|
|
208
|
+
/// idempotent repeat: it changes nothing.
|
|
209
|
+
///
|
|
210
|
+
/// # Errors
|
|
211
|
+
/// [`Refusal::UnknownArm`] past the compiled capacity.
|
|
212
|
+
pub fn close(&mut self, arm: usize) -> Result<Closing, Refusal> {
|
|
213
|
+
let arm = self.arms.get_mut(arm).ok_or(Refusal::UnknownArm)?;
|
|
214
|
+
if !arm.open {
|
|
215
|
+
return Ok(Closing::AlreadyClosed);
|
|
216
|
+
}
|
|
217
|
+
arm.open = false;
|
|
218
|
+
Ok(Closing::Closed)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/// Select by UCB1 and issue a ticket for the chosen arm.
|
|
222
|
+
///
|
|
223
|
+
/// `pick` is consumed only under [`TieBreak::Draw`].
|
|
224
|
+
///
|
|
225
|
+
/// # Errors
|
|
226
|
+
/// [`Refusal::NoOpenArm`], [`Refusal::TicketsFull`] or
|
|
227
|
+
/// [`Refusal::Exhausted`].
|
|
228
|
+
pub fn select_ucb1(&mut self, pick: u64) -> Result<Selection, Refusal> {
|
|
229
|
+
let slot = self.admit()?;
|
|
230
|
+
let (arm, reason) = if let Some(arm) = self.cold(pick) {
|
|
231
|
+
(arm, Reason::ColdStart)
|
|
232
|
+
} else {
|
|
233
|
+
let total = self.total_tries();
|
|
234
|
+
let rule = self.config.tie_break;
|
|
235
|
+
let arm = self.choose(rule, pick, |index, _| self.ucb1_index(index, total));
|
|
236
|
+
(arm.ok_or(Refusal::NoOpenArm)?, Reason::UpperBound)
|
|
237
|
+
};
|
|
238
|
+
self.issue(slot, arm, reason)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// Select by epsilon-greedy and issue a ticket for the chosen arm.
|
|
242
|
+
///
|
|
243
|
+
/// With probability `epsilon` (decided by `draws.gate`) the arm is uniform
|
|
244
|
+
/// over the open arms (by `draws.pick`); otherwise it is the highest exact
|
|
245
|
+
/// mean, ties settled by the configured rule.
|
|
246
|
+
///
|
|
247
|
+
/// # Errors
|
|
248
|
+
/// [`Refusal::NoOpenArm`], [`Refusal::TicketsFull`] or
|
|
249
|
+
/// [`Refusal::Exhausted`].
|
|
250
|
+
pub fn select_epsilon_greedy(
|
|
251
|
+
&mut self,
|
|
252
|
+
epsilon: Ratio,
|
|
253
|
+
draws: Draws,
|
|
254
|
+
) -> Result<Selection, Refusal> {
|
|
255
|
+
let slot = self.admit()?;
|
|
256
|
+
let explore = draw_below(draws.gate, u64::from(epsilon.denominator()))
|
|
257
|
+
< u64::from(epsilon.numerator());
|
|
258
|
+
let (arm, reason) = if let Some(arm) = self.cold(draws.pick) {
|
|
259
|
+
(Some(arm), Reason::ColdStart)
|
|
260
|
+
} else if explore {
|
|
261
|
+
let arm = self.choose(TieBreak::Draw, draws.pick, |_, _| Some(()));
|
|
262
|
+
(arm, Reason::Explore)
|
|
263
|
+
} else {
|
|
264
|
+
let arm = self.choose(self.config.tie_break, draws.pick, |_, arm| {
|
|
265
|
+
Some(self.mean(arm))
|
|
266
|
+
});
|
|
267
|
+
(arm, Reason::Exploit)
|
|
268
|
+
};
|
|
269
|
+
self.issue(slot, arm.ok_or(Refusal::NoOpenArm)?, reason)
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/// UCB1 index of an open arm in Q8.56, in units of the reward range:
|
|
273
|
+
/// `mean / max_reward + sqrt(2 ln(total tries) / tries)`.
|
|
274
|
+
///
|
|
275
|
+
/// `None` for an unknown, closed or cold arm (a cold arm's index is
|
|
276
|
+
/// unbounded; it is selected before any index is compared).
|
|
277
|
+
#[must_use]
|
|
278
|
+
pub fn ucb1_index_q56(&self, arm: usize) -> Option<u64> {
|
|
279
|
+
self.ucb1_index(arm, self.total_tries())
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/// Redeem a ticket with its reward.
|
|
283
|
+
///
|
|
284
|
+
/// # Errors
|
|
285
|
+
/// [`Refusal::UnknownTicket`] if this bandit never issued the ticket
|
|
286
|
+
/// (another scope) or it is spent; [`Refusal::StaleTicket`] if the arm
|
|
287
|
+
/// was reset since the selection;
|
|
288
|
+
/// [`Refusal::RewardOutOfRange`] or [`Refusal::Saturated`] leave the
|
|
289
|
+
/// ticket outstanding so it can be redeemed again.
|
|
290
|
+
pub fn reward(&mut self, ticket: Ticket, reward: u32) -> Result<(), Refusal> {
|
|
291
|
+
self.redeem(ticket, Some(reward))
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/// Give a ticket back without a reward: the selection never happened.
|
|
295
|
+
///
|
|
296
|
+
/// # Errors
|
|
297
|
+
/// [`Refusal::UnknownTicket`] or [`Refusal::StaleTicket`], as for
|
|
298
|
+
/// [`reward`](Self::reward).
|
|
299
|
+
pub fn abandon(&mut self, ticket: Ticket) -> Result<(), Refusal> {
|
|
300
|
+
self.redeem(ticket, None)
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/// Abandon every outstanding ticket whose serial is below `serial`.
|
|
304
|
+
///
|
|
305
|
+
/// `serial` is a cut in this bandit's selection order: a ticket's
|
|
306
|
+
/// [`serial`](Ticket::serial) or an earlier [`next_serial`](Self::next_serial).
|
|
307
|
+
/// Returns how many were revoked. Their late rewards answer
|
|
308
|
+
/// [`Refusal::UnknownTicket`].
|
|
309
|
+
///
|
|
310
|
+
/// # Errors
|
|
311
|
+
/// [`Refusal::UnknownTicket`] for a serial of another scope: it orders
|
|
312
|
+
/// nothing here, and nothing is revoked. No serial is a cut above every
|
|
313
|
+
/// ticket (the last serial of a scope can be outstanding, and `u64::MAX`
|
|
314
|
+
/// belongs to another scope): to revoke them all, call
|
|
315
|
+
/// [`revoke_all`](Self::revoke_all).
|
|
316
|
+
pub fn revoke_before(&mut self, serial: u64) -> Result<usize, Refusal> {
|
|
317
|
+
if scope_of(serial) != self.issuer.scope() {
|
|
318
|
+
return Err(Refusal::UnknownTicket);
|
|
319
|
+
}
|
|
320
|
+
Ok(self.void(|pending| pending.serial < serial))
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/// Abandon every outstanding ticket, whatever its serial. Returns how
|
|
324
|
+
/// many were revoked. Their late rewards answer [`Refusal::UnknownTicket`].
|
|
325
|
+
///
|
|
326
|
+
/// Arm statistics and generations are untouched: unlike
|
|
327
|
+
/// [`reset_all`](Self::reset_all), nothing learned is forgotten. It
|
|
328
|
+
/// cannot refuse, and with nothing outstanding it changes nothing.
|
|
329
|
+
pub fn revoke_all(&mut self) -> usize {
|
|
330
|
+
self.void(|_| true)
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/// Forget one arm: zero its statistics, bump its generation and void its
|
|
334
|
+
/// outstanding tickets. Eligibility is kept. Returns the tickets voided.
|
|
335
|
+
///
|
|
336
|
+
/// # Errors
|
|
337
|
+
/// [`Refusal::UnknownArm`], or [`Refusal::Exhausted`] when the arm has no
|
|
338
|
+
/// generation left.
|
|
339
|
+
pub fn reset(&mut self, arm: usize) -> Result<usize, Refusal> {
|
|
340
|
+
let mut arms = self.arms;
|
|
341
|
+
let state = arms.get_mut(arm).ok_or(Refusal::UnknownArm)?;
|
|
342
|
+
*state = state.forgotten()?;
|
|
343
|
+
Ok(self.replace(arms, |pending| pending.arm == arm))
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/// [`reset`](Self::reset) every arm, or none of them.
|
|
347
|
+
///
|
|
348
|
+
/// # Errors
|
|
349
|
+
/// [`Refusal::Exhausted`] when any arm has no generation left.
|
|
350
|
+
pub fn reset_all(&mut self) -> Result<usize, Refusal> {
|
|
351
|
+
let mut arms = self.arms;
|
|
352
|
+
for arm in &mut arms {
|
|
353
|
+
*arm = arm.forgotten()?;
|
|
354
|
+
}
|
|
355
|
+
Ok(self.replace(arms, |_| true))
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/// Discount one arm's history by `keep`: the reward count becomes
|
|
359
|
+
/// `ceil(count * keep)` and the reward sum `floor(sum * keep)`.
|
|
360
|
+
///
|
|
361
|
+
/// The generation and outstanding tickets are untouched, so late rewards
|
|
362
|
+
/// still land, at full weight. Only `keep` of zero makes an arm cold.
|
|
363
|
+
///
|
|
364
|
+
/// # Errors
|
|
365
|
+
/// [`Refusal::UnknownArm`] past the compiled capacity.
|
|
366
|
+
pub fn decay(&mut self, arm: usize, keep: Ratio) -> Result<(), Refusal> {
|
|
367
|
+
let arm = self.arms.get_mut(arm).ok_or(Refusal::UnknownArm)?;
|
|
368
|
+
discount(arm, keep);
|
|
369
|
+
Ok(())
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/// [`decay`](Self::decay) every arm.
|
|
373
|
+
pub fn decay_all(&mut self, keep: Ratio) {
|
|
374
|
+
for arm in &mut self.arms {
|
|
375
|
+
discount(arm, keep);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/// Everything a selection needs before it may pick: an open arm, a free
|
|
380
|
+
/// ticket slot and a serial left to issue.
|
|
381
|
+
fn admit(&self) -> Result<usize, Refusal> {
|
|
382
|
+
if !self.arms.iter().any(|arm| arm.open) {
|
|
383
|
+
return Err(Refusal::NoOpenArm);
|
|
384
|
+
}
|
|
385
|
+
let slot = self.tickets.iter().position(Option::is_none);
|
|
386
|
+
let slot = slot.ok_or(Refusal::TicketsFull)?;
|
|
387
|
+
match self.issuer.peek() {
|
|
388
|
+
Ok(_) => Ok(slot),
|
|
389
|
+
Err(periscope_machines::Refusal::Exhausted) => Err(Refusal::Exhausted),
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/// The serial is issued last, after every lookup that could fail, so a
|
|
394
|
+
/// refused selection spends no serial and changes nothing.
|
|
395
|
+
fn issue(&mut self, slot: usize, arm: usize, reason: Reason) -> Result<Selection, Refusal> {
|
|
396
|
+
let Some(free) = self.tickets.get_mut(slot).filter(|free| free.is_none()) else {
|
|
397
|
+
debug_assert!(false, "admit answers a free slot below TICKETS");
|
|
398
|
+
return Err(Refusal::TicketsFull);
|
|
399
|
+
};
|
|
400
|
+
let Some(state) = self.arms.get_mut(arm) else {
|
|
401
|
+
debug_assert!(false, "choose answers an arm below ARMS");
|
|
402
|
+
return Err(Refusal::NoOpenArm);
|
|
403
|
+
};
|
|
404
|
+
// An arm holds at most TICKETS outstanding tickets, so this never saturates.
|
|
405
|
+
let pending = state.pending.saturating_add(1);
|
|
406
|
+
let serial = match self.issuer.issue() {
|
|
407
|
+
Ok(serial) => serial,
|
|
408
|
+
Err(periscope_machines::Refusal::Exhausted) => return Err(Refusal::Exhausted),
|
|
409
|
+
};
|
|
410
|
+
*free = Some(Pending { arm, serial });
|
|
411
|
+
state.pending = pending;
|
|
412
|
+
Ok(Selection {
|
|
413
|
+
ticket: Ticket {
|
|
414
|
+
slot,
|
|
415
|
+
arm,
|
|
416
|
+
generation: state.generation,
|
|
417
|
+
serial,
|
|
418
|
+
},
|
|
419
|
+
reason,
|
|
420
|
+
})
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/// Origin first: a ticket this bandit's issuer never handed out names
|
|
424
|
+
/// nothing here, whatever slot, arm and generation it carries. Then the
|
|
425
|
+
/// arm's generation, the slot, and only then the reward; every lookup is
|
|
426
|
+
/// checked, and nothing is written until all of them passed.
|
|
427
|
+
fn redeem(&mut self, ticket: Ticket, reward: Option<u32>) -> Result<(), Refusal> {
|
|
428
|
+
if !self.issuer.owns(ticket.serial) {
|
|
429
|
+
return Err(Refusal::UnknownTicket);
|
|
430
|
+
}
|
|
431
|
+
let arm = self
|
|
432
|
+
.arms
|
|
433
|
+
.get_mut(ticket.arm)
|
|
434
|
+
.ok_or(Refusal::UnknownTicket)?;
|
|
435
|
+
if arm.generation != ticket.generation {
|
|
436
|
+
return Err(Refusal::StaleTicket);
|
|
437
|
+
}
|
|
438
|
+
let expected = Some(Pending {
|
|
439
|
+
arm: ticket.arm,
|
|
440
|
+
serial: ticket.serial,
|
|
441
|
+
});
|
|
442
|
+
let slot = self.tickets.get_mut(ticket.slot);
|
|
443
|
+
let slot = slot
|
|
444
|
+
.filter(|slot| **slot == expected)
|
|
445
|
+
.ok_or(Refusal::UnknownTicket)?;
|
|
446
|
+
let (rewarded, reward_sum) = match reward {
|
|
447
|
+
None => (arm.rewarded, arm.reward_sum),
|
|
448
|
+
Some(reward) if reward > self.config.max_reward => {
|
|
449
|
+
return Err(Refusal::RewardOutOfRange);
|
|
450
|
+
}
|
|
451
|
+
Some(reward) => (
|
|
452
|
+
arm.rewarded.checked_add(1).ok_or(Refusal::Saturated)?,
|
|
453
|
+
arm.reward_sum
|
|
454
|
+
.checked_add(u64::from(reward))
|
|
455
|
+
.ok_or(Refusal::Saturated)?,
|
|
456
|
+
),
|
|
457
|
+
};
|
|
458
|
+
let Some(pending) = arm.pending.checked_sub(1) else {
|
|
459
|
+
debug_assert!(
|
|
460
|
+
false,
|
|
461
|
+
"an outstanding ticket is counted in its arm's pending"
|
|
462
|
+
);
|
|
463
|
+
return Err(Refusal::UnknownTicket);
|
|
464
|
+
};
|
|
465
|
+
*arm = ArmStats {
|
|
466
|
+
rewarded,
|
|
467
|
+
reward_sum,
|
|
468
|
+
pending,
|
|
469
|
+
..*arm
|
|
470
|
+
};
|
|
471
|
+
*slot = None;
|
|
472
|
+
Ok(())
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/// Void the doomed tickets, then install `arms`, the reset arms computed
|
|
476
|
+
/// (and checked) before anything changed.
|
|
477
|
+
fn replace(&mut self, arms: [ArmStats; ARMS], doomed: impl Fn(&Pending) -> bool) -> usize {
|
|
478
|
+
let voided = self.void(doomed);
|
|
479
|
+
self.arms = arms;
|
|
480
|
+
voided
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
fn void(&mut self, doomed: impl Fn(&Pending) -> bool) -> usize {
|
|
484
|
+
let mut voided = 0;
|
|
485
|
+
for slot in &mut self.tickets {
|
|
486
|
+
let Some(pending) = slot.filter(|pending| doomed(pending)) else {
|
|
487
|
+
continue;
|
|
488
|
+
};
|
|
489
|
+
// Voiding cannot refuse, so a broken invariant still frees the slot.
|
|
490
|
+
if let Some(arm) = self.arms.get_mut(pending.arm)
|
|
491
|
+
&& let Some(left) = arm.pending.checked_sub(1)
|
|
492
|
+
{
|
|
493
|
+
arm.pending = left;
|
|
494
|
+
} else {
|
|
495
|
+
debug_assert!(false, "a pending ticket names an arm counting it");
|
|
496
|
+
}
|
|
497
|
+
*slot = None;
|
|
498
|
+
voided += 1;
|
|
499
|
+
}
|
|
500
|
+
voided
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
fn total_tries(&self) -> u64 {
|
|
504
|
+
self.arms
|
|
505
|
+
.iter()
|
|
506
|
+
.filter(|arm| arm.open)
|
|
507
|
+
.fold(0, |total, arm| total.saturating_add(arm.tries()))
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
fn choose<S: Ord + Copy>(
|
|
511
|
+
&self,
|
|
512
|
+
rule: TieBreak,
|
|
513
|
+
pick: u64,
|
|
514
|
+
score: impl Fn(usize, &ArmStats) -> Option<S>,
|
|
515
|
+
) -> Option<usize> {
|
|
516
|
+
best(ARMS, rule, pick, |index| {
|
|
517
|
+
let arm = self.arms.get(index).filter(|arm| arm.open)?;
|
|
518
|
+
Some(Candidate {
|
|
519
|
+
score: score(index, arm)?,
|
|
520
|
+
tries: arm.tries(),
|
|
521
|
+
})
|
|
522
|
+
})
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/// An open arm nobody has tried, if there is one.
|
|
526
|
+
fn cold(&self, pick: u64) -> Option<usize> {
|
|
527
|
+
self.choose(self.config.tie_break, pick, |_, arm| {
|
|
528
|
+
(arm.tries() == 0).then_some(())
|
|
529
|
+
})
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
fn mean(&self, arm: &ArmStats) -> Mean {
|
|
533
|
+
if arm.rewarded == 0 {
|
|
534
|
+
Mean {
|
|
535
|
+
sum: u64::from(self.config.prior),
|
|
536
|
+
count: 1,
|
|
537
|
+
}
|
|
538
|
+
} else {
|
|
539
|
+
Mean {
|
|
540
|
+
sum: arm.reward_sum,
|
|
541
|
+
count: arm.rewarded,
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
fn ucb1_index(&self, arm: usize, total: u64) -> Option<u64> {
|
|
547
|
+
let state = self.arms.get(arm).filter(|arm| arm.open)?;
|
|
548
|
+
if state.tries() == 0 {
|
|
549
|
+
return None;
|
|
550
|
+
}
|
|
551
|
+
let mean = self.mean(state);
|
|
552
|
+
let mean = mean_q56(mean.sum, mean.count, self.config.max_reward);
|
|
553
|
+
// mean <= 2^56 and the bonus is below 2^60 (radicand < 2^63), so no wrap.
|
|
554
|
+
Some(mean + bonus_q56(total, state.tries()))
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
fn discount(arm: &mut ArmStats, keep: Ratio) {
|
|
559
|
+
let numerator = u128::from(keep.numerator());
|
|
560
|
+
let denominator = u128::from(keep.denominator());
|
|
561
|
+
let rewarded = (u128::from(arm.rewarded) * numerator).div_ceil(denominator);
|
|
562
|
+
let reward_sum = u128::from(arm.reward_sum) * numerator / denominator;
|
|
563
|
+
arm.rewarded = u64::try_from(rewarded).unwrap_or(u64::MAX);
|
|
564
|
+
arm.reward_sum = u64::try_from(reward_sum).unwrap_or(u64::MAX);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
#[cfg(test)]
|
|
568
|
+
impl<const ARMS: usize, const TICKETS: usize> Bandit<ARMS, TICKETS> {
|
|
569
|
+
/// Test hook: overwrite one arm's counters to reach `u64` limits.
|
|
570
|
+
pub(crate) fn force(&mut self, arm: usize, generation: u64, rewarded: u64, reward_sum: u64) {
|
|
571
|
+
let arm = &mut self.arms[arm];
|
|
572
|
+
arm.generation = generation;
|
|
573
|
+
arm.rewarded = rewarded;
|
|
574
|
+
arm.reward_sum = reward_sum;
|
|
575
|
+
}
|
|
576
|
+
}
|