chrono_machines 0.5.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +72 -0
- data/Cargo.lock +2 -14
- data/README.md +26 -0
- data/ext/chrono_machines_native/core/Cargo.toml +7 -4
- data/ext/chrono_machines_native/core/src/backoff.rs +58 -92
- data/ext/chrono_machines_native/core/src/dsl.rs +1 -38
- data/ext/chrono_machines_native/core/src/lib.rs +36 -149
- data/ext/chrono_machines_native/core/src/retry.rs +340 -905
- data/ext/chrono_machines_native/core/src/sleep.rs +65 -31
- data/ext/chrono_machines_native/ffi/Cargo.toml +1 -1
- data/ext/chrono_machines_native/ffi/src/lib.rs +3 -68
- data/lib/chrono_machines/configuration.rb +1 -0
- data/lib/chrono_machines/executor.rb +25 -1
- data/lib/chrono_machines/version.rb +1 -1
- metadata +1 -1
|
@@ -16,10 +16,31 @@ use alloc::boxed::Box;
|
|
|
16
16
|
type DefaultRetryBuilder<F, B, T, E> = RetryBuilder<F, B, T, E, fn(&E) -> bool>;
|
|
17
17
|
|
|
18
18
|
/// Type alias for boxed notify callback
|
|
19
|
-
|
|
19
|
+
///
|
|
20
|
+
/// `Send` so that a configured builder — and therefore the future returned by
|
|
21
|
+
/// [`RetryBuilder::call_async`] — can cross threads. Without it `tokio::spawn`
|
|
22
|
+
/// rejects every retry that has a callback attached.
|
|
23
|
+
type NotifyCallback<E> = Box<dyn FnMut(&RetryContext<E>) + Send>;
|
|
20
24
|
|
|
21
25
|
/// Type alias for boxed failure callback
|
|
22
|
-
type FailureCallback<E> = Box<dyn FnMut(&RetryError<E>)>;
|
|
26
|
+
type FailureCallback<E> = Box<dyn FnMut(&RetryError<E>) + Send>;
|
|
27
|
+
|
|
28
|
+
/// Type alias for boxed delay_from hook
|
|
29
|
+
type DelayFromHook<E> = Box<dyn FnMut(&E, u8) -> DelayHint + Send>;
|
|
30
|
+
|
|
31
|
+
/// What a [`delay_from`](RetryBuilder::delay_from) hook wants the loop to do
|
|
32
|
+
/// next. The Rust counterpart of the Ruby executor's nil / Numeric / :halt
|
|
33
|
+
/// return values.
|
|
34
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
35
|
+
pub enum DelayHint {
|
|
36
|
+
/// No opinion - use the backoff strategy.
|
|
37
|
+
Backoff,
|
|
38
|
+
/// Sleep exactly this long, no jitter. Beyond the strategy's max delay
|
|
39
|
+
/// the retry halts with [`RetryErrorKind::HintHalted`].
|
|
40
|
+
Ms(u64),
|
|
41
|
+
/// Stop retrying immediately, failing with the original error.
|
|
42
|
+
Halt,
|
|
43
|
+
}
|
|
23
44
|
|
|
24
45
|
/// Build a terminal [`RetryError`], firing the `on_failure` callback if present.
|
|
25
46
|
fn finalize_failure<E>(
|
|
@@ -30,7 +51,13 @@ fn finalize_failure<E>(
|
|
|
30
51
|
cumulative_delay_ms: u64,
|
|
31
52
|
error: E,
|
|
32
53
|
) -> RetryError<E> {
|
|
33
|
-
let retry_error = RetryError::new(
|
|
54
|
+
let retry_error = RetryError::new(
|
|
55
|
+
kind,
|
|
56
|
+
attempt,
|
|
57
|
+
max_attempts,
|
|
58
|
+
cumulative_delay_ms,
|
|
59
|
+
Some(error),
|
|
60
|
+
);
|
|
34
61
|
if let Some(callback) = on_failure {
|
|
35
62
|
callback(&retry_error);
|
|
36
63
|
}
|
|
@@ -44,6 +71,9 @@ pub enum RetryErrorKind {
|
|
|
44
71
|
Exhausted,
|
|
45
72
|
/// The error was rejected by the `when` predicate.
|
|
46
73
|
PredicateRejected,
|
|
74
|
+
/// A `delay_from` hint halted the retry, either explicitly or by asking
|
|
75
|
+
/// for a delay beyond the strategy's maximum.
|
|
76
|
+
HintHalted,
|
|
47
77
|
}
|
|
48
78
|
|
|
49
79
|
/// Context provided to retry callbacks with observability data.
|
|
@@ -136,6 +166,13 @@ where
|
|
|
136
166
|
RetryErrorKind::PredicateRejected => {
|
|
137
167
|
write!(f, "retry aborted by predicate on attempt {}", self.attempts)?;
|
|
138
168
|
}
|
|
169
|
+
RetryErrorKind::HintHalted => {
|
|
170
|
+
write!(
|
|
171
|
+
f,
|
|
172
|
+
"retry halted by delay_from hint on attempt {}",
|
|
173
|
+
self.attempts
|
|
174
|
+
)?;
|
|
175
|
+
}
|
|
139
176
|
}
|
|
140
177
|
|
|
141
178
|
write!(f, " (cumulative delay {}ms)", self.cumulative_delay_ms)?;
|
|
@@ -234,6 +271,7 @@ where
|
|
|
234
271
|
operation: self,
|
|
235
272
|
backoff,
|
|
236
273
|
when: None,
|
|
274
|
+
delay_from: None,
|
|
237
275
|
notify: None,
|
|
238
276
|
on_success: None,
|
|
239
277
|
on_failure: None,
|
|
@@ -326,7 +364,10 @@ pub trait RetryableExt<T, E>: Retryable<T, E> {
|
|
|
326
364
|
/// let outcome = check_status.with_constant(500).call()?;
|
|
327
365
|
/// # Ok::<(), chrono_machines::RetryError<std::io::Error>>(())
|
|
328
366
|
/// ```
|
|
329
|
-
fn with_constant(
|
|
367
|
+
fn with_constant(
|
|
368
|
+
self,
|
|
369
|
+
delay_ms: u64,
|
|
370
|
+
) -> DefaultRetryBuilder<Self, crate::backoff::ConstantBackoff, T, E>
|
|
330
371
|
where
|
|
331
372
|
Self: Sized,
|
|
332
373
|
{
|
|
@@ -387,6 +428,7 @@ pub struct RetryBuilder<F, B, T, E, W> {
|
|
|
387
428
|
operation: F,
|
|
388
429
|
backoff: B,
|
|
389
430
|
when: Option<W>,
|
|
431
|
+
delay_from: Option<DelayFromHook<E>>,
|
|
390
432
|
notify: Option<NotifyCallback<E>>,
|
|
391
433
|
on_success: Option<NotifyCallback<E>>,
|
|
392
434
|
on_failure: Option<FailureCallback<E>>,
|
|
@@ -394,9 +436,26 @@ pub struct RetryBuilder<F, B, T, E, W> {
|
|
|
394
436
|
_phantom_e: core::marker::PhantomData<E>,
|
|
395
437
|
}
|
|
396
438
|
|
|
439
|
+
/// What the retry loop should do after one attempt.
|
|
440
|
+
///
|
|
441
|
+
/// Produced by [`RetryBuilder::step`], which holds every retry decision —
|
|
442
|
+
/// predicate, exhaustion, delay, callbacks — so the sync and async drivers
|
|
443
|
+
/// share one implementation and can never drift apart. The drivers differ
|
|
444
|
+
/// only in how they await the operation and the sleep.
|
|
445
|
+
enum Step<T, E> {
|
|
446
|
+
/// The attempt succeeded; return this outcome.
|
|
447
|
+
Done(RetryOutcome<T>),
|
|
448
|
+
/// Sleep this many milliseconds, then attempt again.
|
|
449
|
+
Sleep(u64),
|
|
450
|
+
/// Give up and return this error.
|
|
451
|
+
Fail(RetryError<E>),
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Configuration methods, plus the shared decision logic. Deliberately not bound
|
|
455
|
+
// on `F`: the sync driver wants `FnMut() -> Result<T, E>` and the async one
|
|
456
|
+
// wants `FnMut() -> impl Future`, and neither bound belongs on `.when()`.
|
|
397
457
|
impl<F, B, T, E, W> RetryBuilder<F, B, T, E, W>
|
|
398
458
|
where
|
|
399
|
-
F: FnMut() -> Result<T, E>,
|
|
400
459
|
B: BackoffStrategy,
|
|
401
460
|
W: Fn(&E) -> bool,
|
|
402
461
|
{
|
|
@@ -435,6 +494,7 @@ where
|
|
|
435
494
|
operation: self.operation,
|
|
436
495
|
backoff: self.backoff,
|
|
437
496
|
when: Some(predicate),
|
|
497
|
+
delay_from: self.delay_from,
|
|
438
498
|
notify: self.notify,
|
|
439
499
|
on_success: self.on_success,
|
|
440
500
|
on_failure: self.on_failure,
|
|
@@ -443,6 +503,31 @@ where
|
|
|
443
503
|
}
|
|
444
504
|
}
|
|
445
505
|
|
|
506
|
+
/// Let the error dictate the next delay - e.g. HTTP `Retry-After` on 429s.
|
|
507
|
+
///
|
|
508
|
+
/// Called with `(&error, attempt)` before each retry; the returned
|
|
509
|
+
/// [`DelayHint`] overrides the backoff strategy. A [`DelayHint::Ms`]
|
|
510
|
+
/// beyond the strategy's max delay halts with
|
|
511
|
+
/// [`RetryErrorKind::HintHalted`] - the server asked for more patience
|
|
512
|
+
/// than this policy allows.
|
|
513
|
+
///
|
|
514
|
+
/// ```rust,ignore
|
|
515
|
+
/// let outcome = fetch
|
|
516
|
+
/// .retry(ExponentialBackoff::default())
|
|
517
|
+
/// .delay_from(|e: &ApiError, _attempt| match e.retry_after_ms() {
|
|
518
|
+
/// Some(ms) => DelayHint::Ms(ms),
|
|
519
|
+
/// None => DelayHint::Backoff,
|
|
520
|
+
/// })
|
|
521
|
+
/// .call()?;
|
|
522
|
+
/// ```
|
|
523
|
+
pub fn delay_from<C>(mut self, hook: C) -> Self
|
|
524
|
+
where
|
|
525
|
+
C: FnMut(&E, u8) -> DelayHint + Send + 'static,
|
|
526
|
+
{
|
|
527
|
+
self.delay_from = Some(Box::new(hook));
|
|
528
|
+
self
|
|
529
|
+
}
|
|
530
|
+
|
|
446
531
|
/// Add a notification callback that's invoked before each retry
|
|
447
532
|
///
|
|
448
533
|
/// The callback receives a [`RetryContext`] with comprehensive retry state information.
|
|
@@ -475,7 +560,7 @@ where
|
|
|
475
560
|
/// ```
|
|
476
561
|
pub fn notify<C>(mut self, callback: C) -> Self
|
|
477
562
|
where
|
|
478
|
-
C: FnMut(&RetryContext<E>) + 'static,
|
|
563
|
+
C: FnMut(&RetryContext<E>) + Send + 'static,
|
|
479
564
|
{
|
|
480
565
|
self.notify = Some(Box::new(callback));
|
|
481
566
|
self
|
|
@@ -486,7 +571,7 @@ where
|
|
|
486
571
|
/// The callback receives a [`RetryContext`] with no error (error field is None).
|
|
487
572
|
pub fn on_success<C>(mut self, callback: C) -> Self
|
|
488
573
|
where
|
|
489
|
-
C: FnMut(&RetryContext<E>) + 'static,
|
|
574
|
+
C: FnMut(&RetryContext<E>) + Send + 'static,
|
|
490
575
|
{
|
|
491
576
|
self.on_success = Some(Box::new(callback));
|
|
492
577
|
self
|
|
@@ -497,12 +582,137 @@ where
|
|
|
497
582
|
/// The callback receives the rich [`RetryError`] describing the failure.
|
|
498
583
|
pub fn on_failure<C>(mut self, callback: C) -> Self
|
|
499
584
|
where
|
|
500
|
-
C: FnMut(&RetryError<E>) + 'static,
|
|
585
|
+
C: FnMut(&RetryError<E>) + Send + 'static,
|
|
501
586
|
{
|
|
502
587
|
self.on_failure = Some(Box::new(callback));
|
|
503
588
|
self
|
|
504
589
|
}
|
|
505
590
|
|
|
591
|
+
/// Decide what happens after one attempt, firing any callbacks it implies.
|
|
592
|
+
///
|
|
593
|
+
/// This is the whole retry policy. A driver's only jobs are producing the
|
|
594
|
+
/// `Result` and honouring [`Step::Sleep`].
|
|
595
|
+
fn step<R: rand::Rng>(
|
|
596
|
+
&mut self,
|
|
597
|
+
result: Result<T, E>,
|
|
598
|
+
attempt: u8,
|
|
599
|
+
cumulative_delay_ms: u64,
|
|
600
|
+
rng: &mut R,
|
|
601
|
+
) -> Step<T, E> {
|
|
602
|
+
let max_attempts = self.backoff.max_attempts();
|
|
603
|
+
|
|
604
|
+
let error = match result {
|
|
605
|
+
Ok(value) => {
|
|
606
|
+
if let Some(ref mut callback) = self.on_success {
|
|
607
|
+
let ctx = RetryContext {
|
|
608
|
+
attempt,
|
|
609
|
+
next_delay_ms: None,
|
|
610
|
+
cumulative_delay_ms,
|
|
611
|
+
error: None,
|
|
612
|
+
};
|
|
613
|
+
callback(&ctx);
|
|
614
|
+
}
|
|
615
|
+
return Step::Done(RetryOutcome::new(value, attempt, cumulative_delay_ms));
|
|
616
|
+
}
|
|
617
|
+
Err(error) => error,
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
// An error the caller told us not to retry fails immediately, and is
|
|
621
|
+
// reported as rejected rather than exhausted.
|
|
622
|
+
if let Some(ref predicate) = self.when
|
|
623
|
+
&& !predicate(&error)
|
|
624
|
+
{
|
|
625
|
+
return Step::Fail(finalize_failure(
|
|
626
|
+
self.on_failure.as_mut(),
|
|
627
|
+
RetryErrorKind::PredicateRejected,
|
|
628
|
+
attempt,
|
|
629
|
+
max_attempts,
|
|
630
|
+
cumulative_delay_ms,
|
|
631
|
+
error,
|
|
632
|
+
));
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// Out of attempts? Exhaustion wins over any delay hint.
|
|
636
|
+
if !self.backoff.should_retry(attempt) {
|
|
637
|
+
return Step::Fail(finalize_failure(
|
|
638
|
+
self.on_failure.as_mut(),
|
|
639
|
+
RetryErrorKind::Exhausted,
|
|
640
|
+
attempt,
|
|
641
|
+
max_attempts,
|
|
642
|
+
cumulative_delay_ms,
|
|
643
|
+
error,
|
|
644
|
+
));
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
let hint = match self.delay_from {
|
|
648
|
+
Some(ref mut hook) => hook(&error, attempt),
|
|
649
|
+
None => DelayHint::Backoff,
|
|
650
|
+
};
|
|
651
|
+
|
|
652
|
+
let delay_ms = match hint {
|
|
653
|
+
DelayHint::Backoff => {
|
|
654
|
+
// The strategy may still decline to produce a delay.
|
|
655
|
+
let Some(delay_ms) = self.backoff.delay(attempt, rng) else {
|
|
656
|
+
return Step::Fail(finalize_failure(
|
|
657
|
+
self.on_failure.as_mut(),
|
|
658
|
+
RetryErrorKind::Exhausted,
|
|
659
|
+
attempt,
|
|
660
|
+
max_attempts,
|
|
661
|
+
cumulative_delay_ms,
|
|
662
|
+
error,
|
|
663
|
+
));
|
|
664
|
+
};
|
|
665
|
+
delay_ms
|
|
666
|
+
}
|
|
667
|
+
DelayHint::Ms(delay_ms) => {
|
|
668
|
+
if self
|
|
669
|
+
.backoff
|
|
670
|
+
.max_delay_ms()
|
|
671
|
+
.is_some_and(|cap| delay_ms > cap)
|
|
672
|
+
{
|
|
673
|
+
return Step::Fail(finalize_failure(
|
|
674
|
+
self.on_failure.as_mut(),
|
|
675
|
+
RetryErrorKind::HintHalted,
|
|
676
|
+
attempt,
|
|
677
|
+
max_attempts,
|
|
678
|
+
cumulative_delay_ms,
|
|
679
|
+
error,
|
|
680
|
+
));
|
|
681
|
+
}
|
|
682
|
+
delay_ms
|
|
683
|
+
}
|
|
684
|
+
DelayHint::Halt => {
|
|
685
|
+
return Step::Fail(finalize_failure(
|
|
686
|
+
self.on_failure.as_mut(),
|
|
687
|
+
RetryErrorKind::HintHalted,
|
|
688
|
+
attempt,
|
|
689
|
+
max_attempts,
|
|
690
|
+
cumulative_delay_ms,
|
|
691
|
+
error,
|
|
692
|
+
));
|
|
693
|
+
}
|
|
694
|
+
};
|
|
695
|
+
|
|
696
|
+
if let Some(ref mut notify) = self.notify {
|
|
697
|
+
let ctx = RetryContext {
|
|
698
|
+
attempt,
|
|
699
|
+
next_delay_ms: Some(delay_ms),
|
|
700
|
+
cumulative_delay_ms,
|
|
701
|
+
error: Some(&error),
|
|
702
|
+
};
|
|
703
|
+
notify(&ctx);
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
Step::Sleep(delay_ms)
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
impl<F, B, T, E, W> RetryBuilder<F, B, T, E, W>
|
|
711
|
+
where
|
|
712
|
+
F: FnMut() -> Result<T, E>,
|
|
713
|
+
B: BackoffStrategy,
|
|
714
|
+
W: Fn(&E) -> bool,
|
|
715
|
+
{
|
|
506
716
|
/// Execute the retry operation with blocking sleep (requires `std` feature)
|
|
507
717
|
///
|
|
508
718
|
/// Runs the operation synchronously, retrying with blocking sleep between attempts.
|
|
@@ -537,7 +747,9 @@ where
|
|
|
537
747
|
/// Execute the retry operation with a custom sleeper
|
|
538
748
|
///
|
|
539
749
|
/// This low-level method allows providing a custom sleep implementation,
|
|
540
|
-
/// enabling support for
|
|
750
|
+
/// enabling support for embedded systems or testing. The sleeper blocks,
|
|
751
|
+
/// so async runtimes should drive their own loop off
|
|
752
|
+
/// [`Policy::calculate_delay`](crate::Policy::calculate_delay) instead.
|
|
541
753
|
///
|
|
542
754
|
/// # Arguments
|
|
543
755
|
///
|
|
@@ -591,921 +803,144 @@ where
|
|
|
591
803
|
mut rng: R,
|
|
592
804
|
) -> Result<RetryOutcome<T>, RetryError<E>> {
|
|
593
805
|
let mut attempt = 1u8;
|
|
594
|
-
let max_attempts = self.backoff.max_attempts();
|
|
595
806
|
let mut cumulative_delay_ms: u64 = 0;
|
|
596
807
|
|
|
597
808
|
loop {
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
error: None,
|
|
607
|
-
};
|
|
608
|
-
callback(&ctx);
|
|
609
|
-
}
|
|
610
|
-
return Ok(RetryOutcome::new(_value, attempt, cumulative_delay_ms));
|
|
611
|
-
}
|
|
612
|
-
Err(error) => {
|
|
613
|
-
// Check if this error should be retried
|
|
614
|
-
if let Some(ref predicate) = self.when
|
|
615
|
-
&& !predicate(&error) {
|
|
616
|
-
// Error doesn't match predicate, fail immediately
|
|
617
|
-
return Err(finalize_failure(
|
|
618
|
-
self.on_failure.as_mut(),
|
|
619
|
-
RetryErrorKind::PredicateRejected,
|
|
620
|
-
attempt,
|
|
621
|
-
max_attempts,
|
|
622
|
-
cumulative_delay_ms,
|
|
623
|
-
error,
|
|
624
|
-
));
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
// Check if we have retries remaining
|
|
628
|
-
if !self.backoff.should_retry(attempt) {
|
|
629
|
-
return Err(finalize_failure(
|
|
630
|
-
self.on_failure.as_mut(),
|
|
631
|
-
RetryErrorKind::Exhausted,
|
|
632
|
-
attempt,
|
|
633
|
-
max_attempts,
|
|
634
|
-
cumulative_delay_ms,
|
|
635
|
-
error,
|
|
636
|
-
));
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
// Calculate delay
|
|
640
|
-
match self.backoff.delay(attempt, &mut rng) {
|
|
641
|
-
Some(delay_ms) => {
|
|
642
|
-
// Notify if callback is set
|
|
643
|
-
if let Some(ref mut notify) = self.notify {
|
|
644
|
-
let ctx = RetryContext {
|
|
645
|
-
attempt,
|
|
646
|
-
next_delay_ms: Some(delay_ms),
|
|
647
|
-
cumulative_delay_ms,
|
|
648
|
-
error: Some(&error),
|
|
649
|
-
};
|
|
650
|
-
notify(&ctx);
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
// Sleep before retry
|
|
654
|
-
sleeper.sleep_ms(delay_ms);
|
|
655
|
-
cumulative_delay_ms = cumulative_delay_ms.saturating_add(delay_ms);
|
|
656
|
-
attempt = attempt.saturating_add(1);
|
|
657
|
-
}
|
|
658
|
-
None => {
|
|
659
|
-
// Backoff says no more retries
|
|
660
|
-
return Err(finalize_failure(
|
|
661
|
-
self.on_failure.as_mut(),
|
|
662
|
-
RetryErrorKind::Exhausted,
|
|
663
|
-
attempt,
|
|
664
|
-
max_attempts,
|
|
665
|
-
cumulative_delay_ms,
|
|
666
|
-
error,
|
|
667
|
-
));
|
|
668
|
-
}
|
|
669
|
-
}
|
|
809
|
+
let result = (self.operation)();
|
|
810
|
+
match self.step(result, attempt, cumulative_delay_ms, &mut rng) {
|
|
811
|
+
Step::Done(outcome) => return Ok(outcome),
|
|
812
|
+
Step::Fail(error) => return Err(error),
|
|
813
|
+
Step::Sleep(delay_ms) => {
|
|
814
|
+
sleeper.sleep_ms(delay_ms);
|
|
815
|
+
cumulative_delay_ms = cumulative_delay_ms.saturating_add(delay_ms);
|
|
816
|
+
attempt = attempt.saturating_add(1);
|
|
670
817
|
}
|
|
671
818
|
}
|
|
672
819
|
}
|
|
673
820
|
}
|
|
674
821
|
}
|
|
675
822
|
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
#[test]
|
|
704
|
-
fn test_retry_success_after_failures() {
|
|
705
|
-
use core::cell::Cell;
|
|
706
|
-
|
|
707
|
-
let attempts = Cell::new(0);
|
|
708
|
-
|
|
709
|
-
let operation = || {
|
|
710
|
-
let current = attempts.get();
|
|
711
|
-
attempts.set(current + 1);
|
|
712
|
-
|
|
713
|
-
if current < 2 {
|
|
714
|
-
Err(TestError::Retryable)
|
|
715
|
-
} else {
|
|
716
|
-
Ok(42)
|
|
717
|
-
}
|
|
718
|
-
};
|
|
719
|
-
|
|
720
|
-
let result = operation
|
|
721
|
-
.retry(ExponentialBackoff::default().max_attempts(3))
|
|
722
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
723
|
-
|
|
724
|
-
let outcome = result.expect("retry should succeed");
|
|
725
|
-
assert_eq!(outcome.attempts(), 3);
|
|
726
|
-
assert_eq!(outcome.into_inner(), 42);
|
|
727
|
-
assert_eq!(attempts.get(), 3);
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
#[test]
|
|
731
|
-
fn test_retry_exhausted() {
|
|
732
|
-
fn always_fails() -> Result<i32, TestError> {
|
|
733
|
-
Err(TestError::Retryable)
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
let result = always_fails
|
|
737
|
-
.retry(ExponentialBackoff::default().max_attempts(3))
|
|
738
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
739
|
-
|
|
740
|
-
let err = result.expect_err("retry should exhaust");
|
|
741
|
-
assert_eq!(err.kind(), RetryErrorKind::Exhausted);
|
|
742
|
-
assert_eq!(err.attempts(), 3);
|
|
743
|
-
assert_eq!(err.max_attempts(), 3);
|
|
744
|
-
assert!(err.cumulative_delay_ms() > 0);
|
|
745
|
-
if let Some(cause) = err.cause() {
|
|
746
|
-
assert_eq!(cause, &TestError::Retryable);
|
|
747
|
-
} else {
|
|
748
|
-
panic!("expected underlying cause");
|
|
749
|
-
}
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
#[test]
|
|
753
|
-
fn test_retry_when_predicate() {
|
|
754
|
-
fn fails_with_fatal() -> Result<i32, TestError> {
|
|
755
|
-
Err(TestError::Fatal)
|
|
756
|
-
}
|
|
757
|
-
|
|
758
|
-
let result = fails_with_fatal
|
|
759
|
-
.retry(ExponentialBackoff::default())
|
|
760
|
-
.when(|e| matches!(e, TestError::Retryable))
|
|
761
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
762
|
-
|
|
763
|
-
// Fatal error should not be retried
|
|
764
|
-
let err = result.expect_err("retry should stop due to predicate");
|
|
765
|
-
assert_eq!(err.kind(), RetryErrorKind::PredicateRejected);
|
|
766
|
-
if let Some(cause) = err.cause() {
|
|
767
|
-
assert_eq!(cause, &TestError::Fatal);
|
|
768
|
-
} else {
|
|
769
|
-
panic!("expected underlying cause");
|
|
770
|
-
}
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
#[test]
|
|
774
|
-
fn test_retry_notify_callback() {
|
|
775
|
-
use core::cell::{Cell, RefCell};
|
|
776
|
-
#[cfg(feature = "std")]
|
|
777
|
-
use std::rc::Rc;
|
|
778
|
-
|
|
779
|
-
#[cfg(not(feature = "std"))]
|
|
780
|
-
use alloc::rc::Rc;
|
|
781
|
-
|
|
782
|
-
let attempts = Cell::new(0);
|
|
783
|
-
let notify_calls = Rc::new(RefCell::new(Vec::new()));
|
|
784
|
-
let notify_calls_clone = Rc::clone(¬ify_calls);
|
|
785
|
-
|
|
786
|
-
let operation = || {
|
|
787
|
-
let current = attempts.get();
|
|
788
|
-
attempts.set(current + 1);
|
|
789
|
-
|
|
790
|
-
if current < 2 {
|
|
791
|
-
Err(TestError::Retryable)
|
|
792
|
-
} else {
|
|
793
|
-
Ok(42)
|
|
794
|
-
}
|
|
795
|
-
};
|
|
796
|
-
|
|
797
|
-
let result = operation
|
|
798
|
-
.retry(ExponentialBackoff::default().max_attempts(3))
|
|
799
|
-
.notify(move |ctx| {
|
|
800
|
-
// Track notify calls
|
|
801
|
-
notify_calls_clone.borrow_mut().push((
|
|
802
|
-
ctx.attempt,
|
|
803
|
-
ctx.next_delay_ms,
|
|
804
|
-
ctx.cumulative_delay_ms,
|
|
805
|
-
ctx.error.is_some(),
|
|
806
|
-
));
|
|
807
|
-
assert!(ctx.attempt >= 1);
|
|
808
|
-
assert!(ctx.next_delay_ms.is_some());
|
|
809
|
-
assert!(ctx.error.is_some());
|
|
810
|
-
})
|
|
811
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
812
|
-
|
|
813
|
-
let outcome = result.expect("retry should succeed");
|
|
814
|
-
assert_eq!(outcome.attempts(), 3);
|
|
815
|
-
|
|
816
|
-
// Verify notify was called twice (for the two failures)
|
|
817
|
-
let calls = notify_calls.borrow();
|
|
818
|
-
assert_eq!(calls.len(), 2);
|
|
819
|
-
assert_eq!(calls[0].0, 1); // First attempt
|
|
820
|
-
assert_eq!(calls[1].0, 2); // Second attempt
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
#[test]
|
|
824
|
-
fn test_on_success_callback_invoked() {
|
|
825
|
-
use core::cell::Cell;
|
|
826
|
-
use core::sync::atomic::{AtomicUsize, Ordering};
|
|
827
|
-
|
|
828
|
-
static SUCCESS_ATTEMPT: AtomicUsize = AtomicUsize::new(0);
|
|
829
|
-
static SUCCESS_CUMULATIVE_DELAY: AtomicUsize = AtomicUsize::new(0);
|
|
830
|
-
|
|
831
|
-
let attempts = Cell::new(0);
|
|
832
|
-
|
|
833
|
-
let operation = || {
|
|
834
|
-
let current = attempts.get();
|
|
835
|
-
attempts.set(current + 1);
|
|
836
|
-
|
|
837
|
-
if current < 1 {
|
|
838
|
-
Err(TestError::Retryable)
|
|
839
|
-
} else {
|
|
840
|
-
Ok(7)
|
|
841
|
-
}
|
|
842
|
-
};
|
|
843
|
-
|
|
844
|
-
SUCCESS_ATTEMPT.store(0, Ordering::SeqCst);
|
|
845
|
-
SUCCESS_CUMULATIVE_DELAY.store(0, Ordering::SeqCst);
|
|
846
|
-
|
|
847
|
-
let outcome = operation
|
|
848
|
-
.retry(ExponentialBackoff::default().max_attempts(3))
|
|
849
|
-
.on_success(|ctx| {
|
|
850
|
-
SUCCESS_ATTEMPT.store(ctx.attempt as usize, Ordering::SeqCst);
|
|
851
|
-
SUCCESS_CUMULATIVE_DELAY.store(ctx.cumulative_delay_ms as usize, Ordering::SeqCst);
|
|
852
|
-
assert!(ctx.error.is_none());
|
|
853
|
-
assert!(ctx.next_delay_ms.is_none());
|
|
854
|
-
})
|
|
855
|
-
.call_with_sleeper(FnSleeper(|_| {}))
|
|
856
|
-
.expect("retry should succeed");
|
|
857
|
-
|
|
858
|
-
assert_eq!(outcome.into_inner(), 7);
|
|
859
|
-
assert_eq!(SUCCESS_ATTEMPT.load(Ordering::SeqCst), 2);
|
|
860
|
-
// Should have some cumulative delay from the first retry
|
|
861
|
-
assert!(SUCCESS_CUMULATIVE_DELAY.load(Ordering::SeqCst) > 0);
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
#[test]
|
|
865
|
-
fn test_on_failure_callback_invoked() {
|
|
866
|
-
use core::sync::atomic::{AtomicUsize, Ordering};
|
|
867
|
-
|
|
868
|
-
static FAILURE_KIND: AtomicUsize = AtomicUsize::new(0);
|
|
869
|
-
static FAILURE_CUMULATIVE_DELAY: AtomicUsize = AtomicUsize::new(0);
|
|
823
|
+
/// Begin a retry for an operation that returns a future.
|
|
824
|
+
///
|
|
825
|
+
/// The async counterpart to [`Retryable`], and the entry point for
|
|
826
|
+
/// [`RetryBuilder::call_async`]. It is a separate trait because the operation's
|
|
827
|
+
/// bound differs (`FnMut() -> impl Future` rather than `FnMut() -> Result`);
|
|
828
|
+
/// everything after construction — `when`, `notify`, `on_success`,
|
|
829
|
+
/// `on_failure` — is the same builder.
|
|
830
|
+
///
|
|
831
|
+
/// The closure is called once per attempt and must return a *fresh* future each
|
|
832
|
+
/// time, which rules out passing a single future: futures are not restartable.
|
|
833
|
+
///
|
|
834
|
+
/// ```rust,ignore
|
|
835
|
+
/// let outcome = (|| async { fetch().await })
|
|
836
|
+
/// .retry_async(ExponentialBackoff::default())
|
|
837
|
+
/// .when(|e| e.is_transient())
|
|
838
|
+
/// .call_async(|ms| tokio::time::sleep(Duration::from_millis(ms)))
|
|
839
|
+
/// .await?;
|
|
840
|
+
/// ```
|
|
841
|
+
#[cfg(feature = "async")]
|
|
842
|
+
pub trait AsyncRetryable<T, E> {
|
|
843
|
+
/// Begin building a retry operation with the given backoff strategy
|
|
844
|
+
fn retry_async<B: BackoffStrategy>(self, backoff: B) -> DefaultRetryBuilder<Self, B, T, E>
|
|
845
|
+
where
|
|
846
|
+
Self: Sized;
|
|
847
|
+
}
|
|
870
848
|
|
|
871
|
-
|
|
872
|
-
|
|
849
|
+
#[cfg(feature = "async")]
|
|
850
|
+
impl<F, Fut, T, E> AsyncRetryable<T, E> for F
|
|
851
|
+
where
|
|
852
|
+
F: FnMut() -> Fut,
|
|
853
|
+
Fut: core::future::Future<Output = Result<T, E>>,
|
|
854
|
+
{
|
|
855
|
+
fn retry_async<B: BackoffStrategy>(
|
|
856
|
+
self,
|
|
857
|
+
backoff: B,
|
|
858
|
+
) -> RetryBuilder<Self, B, T, E, fn(&E) -> bool> {
|
|
859
|
+
RetryBuilder {
|
|
860
|
+
operation: self,
|
|
861
|
+
backoff,
|
|
862
|
+
when: None,
|
|
863
|
+
delay_from: None,
|
|
864
|
+
notify: None,
|
|
865
|
+
on_success: None,
|
|
866
|
+
on_failure: None,
|
|
867
|
+
_phantom_t: core::marker::PhantomData,
|
|
868
|
+
_phantom_e: core::marker::PhantomData,
|
|
873
869
|
}
|
|
874
|
-
|
|
875
|
-
FAILURE_KIND.store(0, Ordering::SeqCst);
|
|
876
|
-
FAILURE_CUMULATIVE_DELAY.store(0, Ordering::SeqCst);
|
|
877
|
-
|
|
878
|
-
let result = always_fails
|
|
879
|
-
.retry(ExponentialBackoff::default().max_attempts(2))
|
|
880
|
-
.on_failure(|err| {
|
|
881
|
-
let marker = match err.kind() {
|
|
882
|
-
RetryErrorKind::Exhausted => 1,
|
|
883
|
-
RetryErrorKind::PredicateRejected => 2,
|
|
884
|
-
};
|
|
885
|
-
FAILURE_KIND.store(marker, Ordering::SeqCst);
|
|
886
|
-
FAILURE_CUMULATIVE_DELAY.store(err.cumulative_delay_ms() as usize, Ordering::SeqCst);
|
|
887
|
-
})
|
|
888
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
889
|
-
|
|
890
|
-
assert!(result.is_err());
|
|
891
|
-
assert_eq!(FAILURE_KIND.load(Ordering::SeqCst), 1);
|
|
892
|
-
// Should have cumulative delay from retry attempt
|
|
893
|
-
assert!(FAILURE_CUMULATIVE_DELAY.load(Ordering::SeqCst) > 0);
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
#[test]
|
|
897
|
-
fn test_constant_backoff_retry() {
|
|
898
|
-
use core::cell::Cell;
|
|
899
|
-
|
|
900
|
-
let attempts = Cell::new(0);
|
|
901
|
-
|
|
902
|
-
let operation = || {
|
|
903
|
-
let current = attempts.get();
|
|
904
|
-
attempts.set(current + 1);
|
|
905
|
-
|
|
906
|
-
if current < 1 {
|
|
907
|
-
Err(TestError::Retryable)
|
|
908
|
-
} else {
|
|
909
|
-
Ok(42)
|
|
910
|
-
}
|
|
911
|
-
};
|
|
912
|
-
|
|
913
|
-
let result = operation
|
|
914
|
-
.retry(ConstantBackoff::new().delay_ms(10).max_attempts(2))
|
|
915
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
916
|
-
|
|
917
|
-
let outcome = result.expect("retry should succeed");
|
|
918
|
-
assert_eq!(outcome.attempts(), 2);
|
|
919
|
-
assert_eq!(outcome.into_inner(), 42);
|
|
920
|
-
assert_eq!(attempts.get(), 2);
|
|
921
870
|
}
|
|
871
|
+
}
|
|
922
872
|
|
|
873
|
+
#[cfg(feature = "async")]
|
|
874
|
+
impl<F, Fut, B, T, E, W> RetryBuilder<F, B, T, E, W>
|
|
875
|
+
where
|
|
876
|
+
F: FnMut() -> Fut,
|
|
877
|
+
Fut: core::future::Future<Output = Result<T, E>>,
|
|
878
|
+
B: BackoffStrategy,
|
|
879
|
+
W: Fn(&E) -> bool,
|
|
880
|
+
{
|
|
881
|
+
/// Execute the retry operation on an async runtime (requires `std`).
|
|
882
|
+
///
|
|
883
|
+
/// Identical policy to [`call`](Self::call) — same predicate, callbacks and
|
|
884
|
+
/// backoff — but the operation is awaited and the delay yields to the
|
|
885
|
+
/// executor instead of blocking the thread.
|
|
886
|
+
///
|
|
887
|
+
/// The `sleeper` supplies the runtime's timer. Any
|
|
888
|
+
/// `Fn(u64) -> Future<Output = ()>` works, so this crate stays free of a
|
|
889
|
+
/// runtime dependency:
|
|
890
|
+
///
|
|
891
|
+
/// ```rust,ignore
|
|
892
|
+
/// use chrono_machines::{AsyncRetryable, ExponentialBackoff};
|
|
893
|
+
/// use std::time::Duration;
|
|
894
|
+
///
|
|
895
|
+
/// let outcome = (|| async { reqwest::get(url).await })
|
|
896
|
+
/// .retry_async(ExponentialBackoff::default())
|
|
897
|
+
/// .call_async(|ms| tokio::time::sleep(Duration::from_millis(ms)))
|
|
898
|
+
/// .await?;
|
|
899
|
+
/// ```
|
|
900
|
+
///
|
|
901
|
+
/// # Cancellation
|
|
902
|
+
///
|
|
903
|
+
/// Dropping the returned future cancels the retry wherever it stands, in
|
|
904
|
+
/// the operation or mid-sleep. The in-flight attempt is dropped with it, so
|
|
905
|
+
/// the operation must be cancel-safe if that matters to the caller;
|
|
906
|
+
/// `on_failure` does not fire, because nothing failed.
|
|
923
907
|
#[cfg(feature = "std")]
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
let
|
|
929
|
-
|
|
930
|
-
let operation = || {
|
|
931
|
-
let current = attempts.get();
|
|
932
|
-
attempts.set(current + 1);
|
|
933
|
-
|
|
934
|
-
if current < 1 {
|
|
935
|
-
Err(TestError::Retryable)
|
|
936
|
-
} else {
|
|
937
|
-
Ok(42)
|
|
938
|
-
}
|
|
939
|
-
};
|
|
940
|
-
|
|
941
|
-
let start = std::time::Instant::now();
|
|
942
|
-
let result = operation
|
|
943
|
-
.retry(
|
|
944
|
-
ConstantBackoff::new()
|
|
945
|
-
.delay_ms(10)
|
|
946
|
-
.max_attempts(2)
|
|
947
|
-
.jitter_factor(0.0),
|
|
948
|
-
)
|
|
949
|
-
.call();
|
|
950
|
-
|
|
951
|
-
let elapsed = start.elapsed();
|
|
952
|
-
|
|
953
|
-
let outcome = result.expect("retry should succeed");
|
|
954
|
-
assert_eq!(outcome.attempts(), 2);
|
|
955
|
-
assert_eq!(outcome.into_inner(), 42);
|
|
956
|
-
assert!(elapsed.as_millis() >= 9); // At least one 10ms sleep
|
|
957
|
-
}
|
|
958
|
-
|
|
959
|
-
#[test]
|
|
960
|
-
fn test_retry_context_comprehensive() {
|
|
961
|
-
use core::cell::{Cell, RefCell};
|
|
962
|
-
#[cfg(feature = "std")]
|
|
963
|
-
use std::rc::Rc;
|
|
964
|
-
|
|
965
|
-
#[cfg(not(feature = "std"))]
|
|
966
|
-
use alloc::rc::Rc;
|
|
967
|
-
|
|
968
|
-
let attempts = Cell::new(0);
|
|
969
|
-
let notify_contexts = Rc::new(RefCell::new(Vec::new()));
|
|
970
|
-
let notify_contexts_clone = Rc::clone(¬ify_contexts);
|
|
971
|
-
|
|
972
|
-
let operation = || {
|
|
973
|
-
let current = attempts.get();
|
|
974
|
-
attempts.set(current + 1);
|
|
975
|
-
|
|
976
|
-
if current < 3 {
|
|
977
|
-
Err(TestError::Retryable)
|
|
978
|
-
} else {
|
|
979
|
-
Ok(42)
|
|
980
|
-
}
|
|
981
|
-
};
|
|
982
|
-
|
|
983
|
-
let result = operation
|
|
984
|
-
.retry(ConstantBackoff::new().delay_ms(100).max_attempts(5).jitter_factor(0.0))
|
|
985
|
-
.notify(move |ctx| {
|
|
986
|
-
// Capture context for verification
|
|
987
|
-
notify_contexts_clone.borrow_mut().push((
|
|
988
|
-
ctx.attempt,
|
|
989
|
-
ctx.next_delay_ms,
|
|
990
|
-
ctx.cumulative_delay_ms,
|
|
991
|
-
));
|
|
992
|
-
|
|
993
|
-
// Verify error is present during notify
|
|
994
|
-
assert!(ctx.error.is_some());
|
|
995
|
-
if let Some(err) = ctx.error {
|
|
996
|
-
assert_eq!(err, &TestError::Retryable);
|
|
997
|
-
}
|
|
998
|
-
})
|
|
999
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1000
|
-
|
|
1001
|
-
let outcome = result.expect("retry should succeed");
|
|
1002
|
-
assert_eq!(outcome.attempts(), 4);
|
|
1003
|
-
|
|
1004
|
-
// Verify notify contexts
|
|
1005
|
-
let contexts = notify_contexts.borrow();
|
|
1006
|
-
assert_eq!(contexts.len(), 3); // Three failures before success
|
|
1007
|
-
|
|
1008
|
-
// First failure
|
|
1009
|
-
assert_eq!(contexts[0].0, 1);
|
|
1010
|
-
assert_eq!(contexts[0].1, Some(100));
|
|
1011
|
-
assert_eq!(contexts[0].2, 0); // No cumulative delay yet
|
|
1012
|
-
|
|
1013
|
-
// Second failure
|
|
1014
|
-
assert_eq!(contexts[1].0, 2);
|
|
1015
|
-
assert_eq!(contexts[1].1, Some(100));
|
|
1016
|
-
assert_eq!(contexts[1].2, 100); // One delay accumulated
|
|
1017
|
-
|
|
1018
|
-
// Third failure
|
|
1019
|
-
assert_eq!(contexts[2].0, 3);
|
|
1020
|
-
assert_eq!(contexts[2].1, Some(100));
|
|
1021
|
-
assert_eq!(contexts[2].2, 200); // Two delays accumulated
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
#[test]
|
|
1025
|
-
fn test_retry_context_on_success() {
|
|
1026
|
-
use core::cell::{Cell, RefCell};
|
|
1027
|
-
#[cfg(feature = "std")]
|
|
1028
|
-
use std::rc::Rc;
|
|
1029
|
-
|
|
1030
|
-
#[cfg(not(feature = "std"))]
|
|
1031
|
-
use alloc::rc::Rc;
|
|
1032
|
-
|
|
1033
|
-
let attempts = Cell::new(0);
|
|
1034
|
-
let success_context = Rc::new(RefCell::new(None));
|
|
1035
|
-
let success_context_clone = Rc::clone(&success_context);
|
|
1036
|
-
|
|
1037
|
-
let operation = || {
|
|
1038
|
-
let current = attempts.get();
|
|
1039
|
-
attempts.set(current + 1);
|
|
1040
|
-
|
|
1041
|
-
if current < 2 {
|
|
1042
|
-
Err(TestError::Retryable)
|
|
1043
|
-
} else {
|
|
1044
|
-
Ok(100)
|
|
1045
|
-
}
|
|
1046
|
-
};
|
|
1047
|
-
|
|
1048
|
-
let result = operation
|
|
1049
|
-
.retry(ConstantBackoff::new().delay_ms(50).max_attempts(5).jitter_factor(0.0))
|
|
1050
|
-
.on_success(move |ctx| {
|
|
1051
|
-
success_context_clone.borrow_mut().replace((
|
|
1052
|
-
ctx.attempt,
|
|
1053
|
-
ctx.next_delay_ms,
|
|
1054
|
-
ctx.cumulative_delay_ms,
|
|
1055
|
-
ctx.error.is_none(),
|
|
1056
|
-
));
|
|
1057
|
-
})
|
|
1058
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1059
|
-
|
|
1060
|
-
let outcome = result.expect("retry should succeed");
|
|
1061
|
-
assert_eq!(outcome.attempts(), 3);
|
|
1062
|
-
assert_eq!(outcome.cumulative_delay_ms(), 100); // 2 delays of 50ms
|
|
1063
|
-
|
|
1064
|
-
// Verify success context
|
|
1065
|
-
let ctx = success_context.borrow();
|
|
1066
|
-
assert!(ctx.is_some());
|
|
1067
|
-
let (attempt, next_delay, cumulative, no_error) = ctx.unwrap();
|
|
1068
|
-
assert_eq!(attempt, 3);
|
|
1069
|
-
assert_eq!(next_delay, None); // No next delay on success
|
|
1070
|
-
assert_eq!(cumulative, 100);
|
|
1071
|
-
assert!(no_error); // Error should be None
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
#[test]
|
|
1075
|
-
fn test_retry_context_cumulative_accuracy() {
|
|
1076
|
-
use core::cell::{Cell, RefCell};
|
|
1077
|
-
#[cfg(feature = "std")]
|
|
1078
|
-
use std::rc::Rc;
|
|
1079
|
-
|
|
1080
|
-
#[cfg(not(feature = "std"))]
|
|
1081
|
-
use alloc::rc::Rc;
|
|
1082
|
-
|
|
1083
|
-
let attempts = Cell::new(0);
|
|
1084
|
-
let cumulative_progression = Rc::new(RefCell::new(Vec::new()));
|
|
1085
|
-
let cumulative_progression_clone = Rc::clone(&cumulative_progression);
|
|
1086
|
-
|
|
1087
|
-
let operation = || {
|
|
1088
|
-
let current = attempts.get();
|
|
1089
|
-
attempts.set(current + 1);
|
|
1090
|
-
Err::<(), TestError>(TestError::Retryable)
|
|
1091
|
-
};
|
|
1092
|
-
|
|
1093
|
-
let _result = operation
|
|
1094
|
-
.retry(ConstantBackoff::new().delay_ms(25).max_attempts(4).jitter_factor(0.0))
|
|
1095
|
-
.notify(move |ctx| {
|
|
1096
|
-
cumulative_progression_clone.borrow_mut().push(ctx.cumulative_delay_ms);
|
|
1097
|
-
})
|
|
1098
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1099
|
-
|
|
1100
|
-
// Verify cumulative delay progression
|
|
1101
|
-
let progression = cumulative_progression.borrow();
|
|
1102
|
-
assert_eq!(progression.len(), 3); // 3 retries before exhaustion
|
|
1103
|
-
assert_eq!(progression[0], 0); // Before first sleep
|
|
1104
|
-
assert_eq!(progression[1], 25); // After first sleep
|
|
1105
|
-
assert_eq!(progression[2], 50); // After second sleep
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
// ============================================================================
|
|
1109
|
-
// RetryableExt Tests
|
|
1110
|
-
// ============================================================================
|
|
1111
|
-
|
|
1112
|
-
#[test]
|
|
1113
|
-
fn test_with_exponential_success() {
|
|
1114
|
-
use super::RetryableExt;
|
|
1115
|
-
|
|
1116
|
-
fn always_succeeds() -> Result<i32, TestError> {
|
|
1117
|
-
Ok(42)
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
let result = always_succeeds
|
|
1121
|
-
.with_exponential()
|
|
1122
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1123
|
-
|
|
1124
|
-
let outcome = result.expect("retry should succeed");
|
|
1125
|
-
assert_eq!(outcome.attempts(), 1);
|
|
1126
|
-
assert_eq!(outcome.into_inner(), 42);
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
#[test]
|
|
1130
|
-
fn test_with_exponential_retry_behavior() {
|
|
1131
|
-
use super::RetryableExt;
|
|
1132
|
-
use core::cell::Cell;
|
|
1133
|
-
|
|
1134
|
-
let attempts = Cell::new(0);
|
|
1135
|
-
|
|
1136
|
-
let operation = || {
|
|
1137
|
-
let current = attempts.get();
|
|
1138
|
-
attempts.set(current + 1);
|
|
1139
|
-
|
|
1140
|
-
if current < 2 {
|
|
1141
|
-
Err(TestError::Retryable)
|
|
1142
|
-
} else {
|
|
1143
|
-
Ok(100)
|
|
1144
|
-
}
|
|
1145
|
-
};
|
|
1146
|
-
|
|
1147
|
-
let result = operation
|
|
1148
|
-
.with_exponential()
|
|
1149
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1150
|
-
|
|
1151
|
-
let outcome = result.expect("retry should succeed");
|
|
1152
|
-
assert_eq!(outcome.attempts(), 3);
|
|
1153
|
-
assert_eq!(outcome.into_inner(), 100);
|
|
1154
|
-
// Verify default max_attempts was used (3)
|
|
1155
|
-
assert_eq!(attempts.get(), 3);
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
#[test]
|
|
1159
|
-
fn test_with_exponential_exhausted() {
|
|
1160
|
-
use super::RetryableExt;
|
|
1161
|
-
|
|
1162
|
-
fn always_fails() -> Result<i32, TestError> {
|
|
1163
|
-
Err(TestError::Retryable)
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
let result = always_fails
|
|
1167
|
-
.with_exponential()
|
|
1168
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1169
|
-
|
|
1170
|
-
let err = result.expect_err("retry should exhaust");
|
|
1171
|
-
assert_eq!(err.kind(), RetryErrorKind::Exhausted);
|
|
1172
|
-
assert_eq!(err.attempts(), 3); // Default max_attempts
|
|
1173
|
-
assert_eq!(err.max_attempts(), 3);
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
#[test]
|
|
1177
|
-
fn test_with_exponential_chaining() {
|
|
1178
|
-
use super::RetryableExt;
|
|
1179
|
-
use core::cell::Cell;
|
|
1180
|
-
|
|
1181
|
-
let attempts = Cell::new(0);
|
|
1182
|
-
|
|
1183
|
-
let operation = || {
|
|
1184
|
-
let current = attempts.get();
|
|
1185
|
-
attempts.set(current + 1);
|
|
1186
|
-
|
|
1187
|
-
if current < 1 {
|
|
1188
|
-
Err(TestError::Retryable)
|
|
1189
|
-
} else {
|
|
1190
|
-
Ok(7)
|
|
1191
|
-
}
|
|
1192
|
-
};
|
|
1193
|
-
|
|
1194
|
-
// Test that .with_exponential() can be chained with other builder methods
|
|
1195
|
-
let result = operation
|
|
1196
|
-
.with_exponential()
|
|
1197
|
-
.when(|e| matches!(e, TestError::Retryable))
|
|
1198
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1199
|
-
|
|
1200
|
-
let outcome = result.expect("retry should succeed");
|
|
1201
|
-
assert_eq!(outcome.attempts(), 2);
|
|
1202
|
-
assert_eq!(outcome.into_inner(), 7);
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
#[test]
|
|
1206
|
-
fn test_with_constant_success() {
|
|
1207
|
-
use super::RetryableExt;
|
|
1208
|
-
|
|
1209
|
-
fn always_succeeds() -> Result<i32, TestError> {
|
|
1210
|
-
Ok(99)
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
let result = always_succeeds
|
|
1214
|
-
.with_constant(250)
|
|
1215
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1216
|
-
|
|
1217
|
-
let outcome = result.expect("retry should succeed");
|
|
1218
|
-
assert_eq!(outcome.attempts(), 1);
|
|
1219
|
-
assert_eq!(outcome.into_inner(), 99);
|
|
908
|
+
pub async fn call_async<S: crate::sleep::AsyncSleeper>(
|
|
909
|
+
self,
|
|
910
|
+
sleeper: S,
|
|
911
|
+
) -> Result<RetryOutcome<T>, RetryError<E>> {
|
|
912
|
+
let rng: StdRng = rand::make_rng();
|
|
913
|
+
self.call_async_with_rng(sleeper, rng).await
|
|
1220
914
|
}
|
|
1221
915
|
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
let
|
|
1234
|
-
let
|
|
1235
|
-
|
|
1236
|
-
let operation = || {
|
|
1237
|
-
let current = attempts.get();
|
|
1238
|
-
attempts.set(current + 1);
|
|
1239
|
-
|
|
1240
|
-
if current < 2 {
|
|
1241
|
-
Err(TestError::Retryable)
|
|
1242
|
-
} else {
|
|
1243
|
-
Ok(42)
|
|
1244
|
-
}
|
|
1245
|
-
};
|
|
916
|
+
/// Execute the retry operation on an async runtime with a caller-supplied RNG.
|
|
917
|
+
///
|
|
918
|
+
/// The `no_std` entry point, mirroring
|
|
919
|
+
/// [`call_with_sleeper_and_rng`](Self::call_with_sleeper_and_rng): callers
|
|
920
|
+
/// provide their own [`rand::Rng`] for jitter instead of relying on the
|
|
921
|
+
/// `std`-only `make_rng`.
|
|
922
|
+
pub async fn call_async_with_rng<S: crate::sleep::AsyncSleeper, R: rand::Rng>(
|
|
923
|
+
mut self,
|
|
924
|
+
sleeper: S,
|
|
925
|
+
mut rng: R,
|
|
926
|
+
) -> Result<RetryOutcome<T>, RetryError<E>> {
|
|
927
|
+
let mut attempt = 1u8;
|
|
928
|
+
let mut cumulative_delay_ms: u64 = 0;
|
|
1246
929
|
|
|
1247
|
-
|
|
1248
|
-
.
|
|
1249
|
-
.
|
|
1250
|
-
|
|
1251
|
-
|
|
930
|
+
loop {
|
|
931
|
+
let result = (self.operation)().await;
|
|
932
|
+
match self.step(result, attempt, cumulative_delay_ms, &mut rng) {
|
|
933
|
+
Step::Done(outcome) => return Ok(outcome),
|
|
934
|
+
Step::Fail(error) => return Err(error),
|
|
935
|
+
Step::Sleep(delay_ms) => {
|
|
936
|
+
sleeper.sleep_ms(delay_ms).await;
|
|
937
|
+
cumulative_delay_ms = cumulative_delay_ms.saturating_add(delay_ms);
|
|
938
|
+
attempt = attempt.saturating_add(1);
|
|
1252
939
|
}
|
|
1253
|
-
})
|
|
1254
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1255
|
-
|
|
1256
|
-
let outcome = result.expect("retry should succeed");
|
|
1257
|
-
assert_eq!(outcome.attempts(), 3);
|
|
1258
|
-
|
|
1259
|
-
// Verify constant delay was used (ConstantBackoff default has no jitter)
|
|
1260
|
-
let recorded_delays = delays.borrow();
|
|
1261
|
-
assert_eq!(recorded_delays.len(), 2); // Two retries
|
|
1262
|
-
assert_eq!(recorded_delays[0], 500);
|
|
1263
|
-
assert_eq!(recorded_delays[1], 500);
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
#[test]
|
|
1267
|
-
fn test_with_constant_chaining() {
|
|
1268
|
-
use super::RetryableExt;
|
|
1269
|
-
use core::cell::Cell;
|
|
1270
|
-
|
|
1271
|
-
let attempts = Cell::new(0);
|
|
1272
|
-
|
|
1273
|
-
let operation = || {
|
|
1274
|
-
let current = attempts.get();
|
|
1275
|
-
attempts.set(current + 1);
|
|
1276
|
-
|
|
1277
|
-
if current < 1 {
|
|
1278
|
-
Err(TestError::Fatal)
|
|
1279
|
-
} else {
|
|
1280
|
-
Ok(123)
|
|
1281
940
|
}
|
|
1282
|
-
};
|
|
1283
|
-
|
|
1284
|
-
// Test that .with_constant() can be chained with predicate
|
|
1285
|
-
let result = operation
|
|
1286
|
-
.with_constant(100)
|
|
1287
|
-
.when(|e| matches!(e, TestError::Retryable))
|
|
1288
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1289
|
-
|
|
1290
|
-
// Should fail immediately due to predicate
|
|
1291
|
-
let err = result.expect_err("retry should fail due to predicate");
|
|
1292
|
-
assert_eq!(err.kind(), RetryErrorKind::PredicateRejected);
|
|
1293
|
-
assert_eq!(err.attempts(), 1);
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
#[test]
|
|
1297
|
-
fn test_with_fibonacci_success() {
|
|
1298
|
-
use super::RetryableExt;
|
|
1299
|
-
|
|
1300
|
-
fn always_succeeds() -> Result<String, TestError> {
|
|
1301
|
-
Ok("success".to_string())
|
|
1302
941
|
}
|
|
1303
|
-
|
|
1304
|
-
let result = always_succeeds
|
|
1305
|
-
.with_fibonacci()
|
|
1306
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1307
|
-
|
|
1308
|
-
let outcome = result.expect("retry should succeed");
|
|
1309
|
-
assert_eq!(outcome.attempts(), 1);
|
|
1310
|
-
assert_eq!(outcome.into_inner(), "success");
|
|
1311
|
-
}
|
|
1312
|
-
|
|
1313
|
-
#[test]
|
|
1314
|
-
fn test_with_fibonacci_retry_behavior() {
|
|
1315
|
-
use super::RetryableExt;
|
|
1316
|
-
use core::cell::Cell;
|
|
1317
|
-
|
|
1318
|
-
let attempts = Cell::new(0);
|
|
1319
|
-
|
|
1320
|
-
let operation = || {
|
|
1321
|
-
let current = attempts.get();
|
|
1322
|
-
attempts.set(current + 1);
|
|
1323
|
-
|
|
1324
|
-
if current < 3 {
|
|
1325
|
-
Err(TestError::Retryable)
|
|
1326
|
-
} else {
|
|
1327
|
-
Ok(777)
|
|
1328
|
-
}
|
|
1329
|
-
};
|
|
1330
|
-
|
|
1331
|
-
let result = operation
|
|
1332
|
-
.with_fibonacci()
|
|
1333
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1334
|
-
|
|
1335
|
-
let outcome = result.expect("retry should succeed");
|
|
1336
|
-
assert_eq!(outcome.attempts(), 4);
|
|
1337
|
-
assert_eq!(outcome.into_inner(), 777);
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
#[test]
|
|
1341
|
-
fn test_with_fibonacci_exhausted() {
|
|
1342
|
-
use super::RetryableExt;
|
|
1343
|
-
|
|
1344
|
-
fn always_fails() -> Result<i32, TestError> {
|
|
1345
|
-
Err(TestError::Retryable)
|
|
1346
|
-
}
|
|
1347
|
-
|
|
1348
|
-
let result = always_fails
|
|
1349
|
-
.with_fibonacci()
|
|
1350
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1351
|
-
|
|
1352
|
-
let err = result.expect_err("retry should exhaust");
|
|
1353
|
-
assert_eq!(err.kind(), RetryErrorKind::Exhausted);
|
|
1354
|
-
assert_eq!(err.attempts(), 8); // Fibonacci default max_attempts
|
|
1355
|
-
assert_eq!(err.max_attempts(), 8);
|
|
1356
|
-
}
|
|
1357
|
-
|
|
1358
|
-
#[test]
|
|
1359
|
-
fn test_with_fibonacci_chaining() {
|
|
1360
|
-
use super::RetryableExt;
|
|
1361
|
-
use core::cell::Cell;
|
|
1362
|
-
use core::sync::atomic::{AtomicUsize, Ordering};
|
|
1363
|
-
|
|
1364
|
-
static SUCCESS_COUNT: AtomicUsize = AtomicUsize::new(0);
|
|
1365
|
-
let attempts = Cell::new(0);
|
|
1366
|
-
|
|
1367
|
-
let operation = || {
|
|
1368
|
-
let current = attempts.get();
|
|
1369
|
-
attempts.set(current + 1);
|
|
1370
|
-
|
|
1371
|
-
if current < 2 {
|
|
1372
|
-
Err(TestError::Retryable)
|
|
1373
|
-
} else {
|
|
1374
|
-
Ok(555)
|
|
1375
|
-
}
|
|
1376
|
-
};
|
|
1377
|
-
|
|
1378
|
-
SUCCESS_COUNT.store(0, Ordering::SeqCst);
|
|
1379
|
-
|
|
1380
|
-
// Test that .with_fibonacci() can be chained with callbacks
|
|
1381
|
-
let result = operation
|
|
1382
|
-
.with_fibonacci()
|
|
1383
|
-
.on_success(|_ctx| {
|
|
1384
|
-
SUCCESS_COUNT.fetch_add(1, Ordering::SeqCst);
|
|
1385
|
-
})
|
|
1386
|
-
.call_with_sleeper(FnSleeper(|_| {}));
|
|
1387
|
-
|
|
1388
|
-
let outcome = result.expect("retry should succeed");
|
|
1389
|
-
assert_eq!(outcome.attempts(), 3);
|
|
1390
|
-
assert_eq!(outcome.into_inner(), 555);
|
|
1391
|
-
assert_eq!(SUCCESS_COUNT.load(Ordering::SeqCst), 1);
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
#[test]
|
|
1395
|
-
fn test_all_extension_methods_produce_working_retries() {
|
|
1396
|
-
use super::RetryableExt;
|
|
1397
|
-
use core::cell::Cell;
|
|
1398
|
-
|
|
1399
|
-
let attempts_exp = Cell::new(0);
|
|
1400
|
-
let op_exp = || {
|
|
1401
|
-
let current = attempts_exp.get();
|
|
1402
|
-
attempts_exp.set(current + 1);
|
|
1403
|
-
if current < 1 {
|
|
1404
|
-
Err(TestError::Retryable)
|
|
1405
|
-
} else {
|
|
1406
|
-
Ok(1)
|
|
1407
|
-
}
|
|
1408
|
-
};
|
|
1409
|
-
|
|
1410
|
-
let attempts_const = Cell::new(0);
|
|
1411
|
-
let op_const = || {
|
|
1412
|
-
let current = attempts_const.get();
|
|
1413
|
-
attempts_const.set(current + 1);
|
|
1414
|
-
if current < 1 {
|
|
1415
|
-
Err(TestError::Retryable)
|
|
1416
|
-
} else {
|
|
1417
|
-
Ok(2)
|
|
1418
|
-
}
|
|
1419
|
-
};
|
|
1420
|
-
|
|
1421
|
-
let attempts_fib = Cell::new(0);
|
|
1422
|
-
let op_fib = || {
|
|
1423
|
-
let current = attempts_fib.get();
|
|
1424
|
-
attempts_fib.set(current + 1);
|
|
1425
|
-
if current < 1 {
|
|
1426
|
-
Err(TestError::Retryable)
|
|
1427
|
-
} else {
|
|
1428
|
-
Ok(3)
|
|
1429
|
-
}
|
|
1430
|
-
};
|
|
1431
|
-
|
|
1432
|
-
// All extension methods should produce working retry builders
|
|
1433
|
-
let r1 = op_exp
|
|
1434
|
-
.with_exponential()
|
|
1435
|
-
.call_with_sleeper(FnSleeper(|_| {}))
|
|
1436
|
-
.expect("exponential retry works");
|
|
1437
|
-
|
|
1438
|
-
let r2 = op_const
|
|
1439
|
-
.with_constant(100)
|
|
1440
|
-
.call_with_sleeper(FnSleeper(|_| {}))
|
|
1441
|
-
.expect("constant retry works");
|
|
1442
|
-
|
|
1443
|
-
let r3 = op_fib
|
|
1444
|
-
.with_fibonacci()
|
|
1445
|
-
.call_with_sleeper(FnSleeper(|_| {}))
|
|
1446
|
-
.expect("fibonacci retry works");
|
|
1447
|
-
|
|
1448
|
-
assert_eq!(r1.into_inner(), 1);
|
|
1449
|
-
assert_eq!(r2.into_inner(), 2);
|
|
1450
|
-
assert_eq!(r3.into_inner(), 3);
|
|
1451
|
-
}
|
|
1452
|
-
|
|
1453
|
-
#[cfg(feature = "std")]
|
|
1454
|
-
#[test]
|
|
1455
|
-
fn test_with_exponential_std_sleeper() {
|
|
1456
|
-
use super::RetryableExt;
|
|
1457
|
-
use core::cell::Cell;
|
|
1458
|
-
|
|
1459
|
-
let attempts = Cell::new(0);
|
|
1460
|
-
|
|
1461
|
-
let operation = || {
|
|
1462
|
-
let current = attempts.get();
|
|
1463
|
-
attempts.set(current + 1);
|
|
1464
|
-
|
|
1465
|
-
if current < 1 {
|
|
1466
|
-
Err(TestError::Retryable)
|
|
1467
|
-
} else {
|
|
1468
|
-
Ok(999)
|
|
1469
|
-
}
|
|
1470
|
-
};
|
|
1471
|
-
|
|
1472
|
-
// Test with std sleeper
|
|
1473
|
-
let result = operation.with_exponential().call();
|
|
1474
|
-
|
|
1475
|
-
let outcome = result.expect("retry should succeed");
|
|
1476
|
-
assert_eq!(outcome.attempts(), 2);
|
|
1477
|
-
assert_eq!(outcome.into_inner(), 999);
|
|
1478
|
-
}
|
|
1479
|
-
|
|
1480
|
-
#[cfg(feature = "std")]
|
|
1481
|
-
#[test]
|
|
1482
|
-
fn test_with_constant_std_sleeper() {
|
|
1483
|
-
use super::RetryableExt;
|
|
1484
|
-
|
|
1485
|
-
fn always_succeeds() -> Result<i32, TestError> {
|
|
1486
|
-
Ok(888)
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
// Test with std sleeper
|
|
1490
|
-
let result = always_succeeds.with_constant(50).call();
|
|
1491
|
-
|
|
1492
|
-
let outcome = result.expect("retry should succeed");
|
|
1493
|
-
assert_eq!(outcome.into_inner(), 888);
|
|
1494
|
-
}
|
|
1495
|
-
|
|
1496
|
-
#[cfg(feature = "std")]
|
|
1497
|
-
#[test]
|
|
1498
|
-
fn test_with_fibonacci_std_sleeper() {
|
|
1499
|
-
use super::RetryableExt;
|
|
1500
|
-
|
|
1501
|
-
fn always_succeeds() -> Result<i32, TestError> {
|
|
1502
|
-
Ok(444)
|
|
1503
|
-
}
|
|
1504
|
-
|
|
1505
|
-
// Test with std sleeper
|
|
1506
|
-
let result = always_succeeds.with_fibonacci().call();
|
|
1507
|
-
|
|
1508
|
-
let outcome = result.expect("retry should succeed");
|
|
1509
|
-
assert_eq!(outcome.into_inner(), 444);
|
|
1510
942
|
}
|
|
1511
943
|
}
|
|
944
|
+
|
|
945
|
+
#[cfg(test)]
|
|
946
|
+
mod tests;
|