@temporalio/core-bridge 0.16.0 → 0.17.1

package/sdk-core/src/worker/activities.rs

@@ -10,7 +10,6 @@ use activity_heartbeat_manager::ActivityHeartbeatManager;
 use dashmap::DashMap;
 use std::{
     convert::TryInto,
-    ops::Div,
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -80,6 +79,9 @@ pub(crate) struct WorkerActivityTasks {
     activities_semaphore: Semaphore,
 
     metrics: MetricsContext,
+
+    max_heartbeat_throttle_interval: Duration,
+    default_heartbeat_throttle_interval: Duration,
 }
 
 impl WorkerActivityTasks {
@@ -88,6 +90,8 @@ impl WorkerActivityTasks {
         poller: BoxedActPoller,
         sg: Arc<impl ServerGatewayApis + Send + Sync + 'static + ?Sized>,
         metrics: MetricsContext,
+        max_heartbeat_throttle_interval: Duration,
+        default_heartbeat_throttle_interval: Duration,
     ) -> Self {
         Self {
             heartbeat_manager: ActivityHeartbeatManager::new(sg),
@@ -95,12 +99,13 @@ impl WorkerActivityTasks {
             poller,
             activities_semaphore: Semaphore::new(max_activity_tasks),
             metrics,
+            max_heartbeat_throttle_interval,
+            default_heartbeat_throttle_interval,
         }
     }
 
     pub(crate) fn notify_shutdown(&self) {
         self.poller.notify_shutdown();
-        self.heartbeat_manager.notify_shutdown();
     }
 
     pub(crate) async fn shutdown(self) {
@@ -171,16 +176,19 @@ impl WorkerActivityTasks {
         status: activity_result::Status,
         gateway: &(dyn ServerGatewayApis + Send + Sync),
     ) -> Result<(), CompleteActivityError> {
-        if let Some(act_info) = self.outstanding_activity_tasks.get(&task_token) {
+        if let Some((_, act_info)) = self.outstanding_activity_tasks.remove(&task_token) {
             let act_metrics = self.metrics.with_new_attrs([
                 activity_type(act_info.activity_type.clone()),
                 workflow_type(act_info.workflow_type.clone()),
             ]);
             act_metrics.act_execution_latency(act_info.start_time.elapsed());
+            self.activities_semaphore.add_permits(1);
+            self.heartbeat_manager.evict(task_token.clone());
+            let known_not_found = act_info.known_not_found;
+            drop(act_info); // TODO: Get rid of dashmap. If we hold ref across await, bad stuff.
 
             // No need to report activities which we already know the server doesn't care about
-            let should_remove = if !act_info.known_not_found {
-                drop(act_info); // TODO: Get rid of dashmap. If we hold ref across await, bad stuff.
+            if !known_not_found {
                 let maybe_net_err = match status {
                     activity_result::Status::WillCompleteAsync(_) => None,
                     activity_result::Status::Completed(ar::Success { result }) => gateway
@@ -195,19 +203,17 @@ impl WorkerActivityTasks {
                             .err()
                     }
                     activity_result::Status::Cancelled(ar::Cancellation { failure }) => {
-                        let details = match failure {
-                            Some(Failure {
-                                failure_info:
-                                    Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo {
-                                        details,
-                                    })),
-                                ..
-                            }) => details,
-                            _ => {
-                                warn!(task_token = ? task_token,
-                                    "Expected activity cancelled status with CanceledFailureInfo");
-                                None
-                            }
+                        let details = if let Some(Failure {
+                            failure_info:
+                                Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo { details })),
+                            ..
+                        }) = failure
+                        {
+                            details
+                        } else {
+                            warn!(task_token = ? task_token,
+                                "Expected activity cancelled status with CanceledFailureInfo");
+                            None
                         };
                         gateway
                             .cancel_activity_task(task_token.clone(), details.map(Into::into))
@@ -215,37 +221,24 @@ impl WorkerActivityTasks {
                             .err()
                     }
                 };
-                match maybe_net_err {
-                    Some(e) if e.code() == tonic::Code::NotFound => {
+
+                if let Some(e) = maybe_net_err {
+                    if e.code() == tonic::Code::NotFound {
                         warn!(task_token = ?task_token, details = ?e, "Activity not found on \
                         completion. This may happen if the activity has already been cancelled but \
                         completed anyway.");
-                        true
-                    }
-                    Some(err) => return Err(err.into()),
-                    None => true,
-                }
-            } else {
-                true
+                    } else {
+                        return Err(e.into());
+                    };
+                };
             };
-
-            if should_remove
-                && self
-                    .outstanding_activity_tasks
-                    .remove(&task_token)
-                    .is_some()
-            {
-                self.activities_semaphore.add_permits(1);
-                self.heartbeat_manager.evict(task_token);
-            }
-            Ok(())
         } else {
             warn!(
                 "Attempted to complete activity task {} but we were not tracking it",
                 &task_token
             );
-            Ok(())
         }
+        Ok(())
     }
 
     /// Attempt to record an activity heartbeat
@@ -254,22 +247,30 @@ impl WorkerActivityTasks {
         details: ActivityHeartbeat,
     ) -> Result<(), ActivityHeartbeatError> {
         // TODO: Propagate these back as cancels. Silent fails is too nonobvious
-        let t: Duration = self
+        let heartbeat_timeout: Duration = self
             .outstanding_activity_tasks
             .get(&TaskToken(details.task_token.clone()))
             .ok_or(ActivityHeartbeatError::UnknownActivity)?
             .heartbeat_timeout
             .clone()
-            .ok_or(ActivityHeartbeatError::HeartbeatTimeoutNotSet)?
+            // We treat None as 0 (even though heartbeat_timeout is never set to None by the server)
+            .unwrap_or_default()
             .try_into()
+            // This technically should never happen since prost duration should be directly mappable
+            // to std::time::Duration.
             .or(Err(ActivityHeartbeatError::InvalidHeartbeatTimeout))?;
+
         // There is a bug in the server that translates non-set heartbeat timeouts into 0 duration.
         // That's why we treat 0 the same way as None, otherwise we wouldn't know which aggregation
         // delay to use, and using 0 is not a good idea as SDK would hammer the server too hard.
-        if t.as_millis() == 0 {
-            return Err(ActivityHeartbeatError::HeartbeatTimeoutNotSet);
-        }
-        self.heartbeat_manager.record(details, t.div(2))
+        let throttle_interval = if heartbeat_timeout.as_millis() == 0 {
+            self.default_heartbeat_throttle_interval
+        } else {
+            heartbeat_timeout.mul_f64(0.8)
+        };
+        let throttle_interval =
+            std::cmp::min(throttle_interval, self.max_heartbeat_throttle_interval);
+        self.heartbeat_manager.record(details, throttle_interval)
     }
 
     async fn next_pending_cancel_task(&self) -> Result<Option<ActivityTask>, PollActivityError> {

package/sdk-core/src/worker/config.rs

@@ -51,6 +51,17 @@ pub struct WorkerConfig {
     /// and moved to the non-sticky queue where it may be picked up by any worker.
     #[builder(default = "Duration::from_secs(10)")]
     pub sticky_queue_schedule_to_start_timeout: Duration,
+
+    /// Longest interval for throttling activity heartbeats
+    #[builder(default = "Duration::from_secs(60)")]
+    pub max_heartbeat_throttle_interval: Duration,
+
+    /// Default interval for throttling activity heartbeats in case
+    /// `ActivityOptions.heartbeat_timeout` is unset.
+    /// When the timeout *is* set in the `ActivityOptions`, throttling is set to
+    /// `heartbeat_timeout * 0.8`.
+    #[builder(default = "Duration::from_secs(30)")]
+    pub default_heartbeat_throttle_interval: Duration,
 }
 
 impl WorkerConfigBuilder {

package/sdk-core/src/worker/dispatcher.rs

@@ -6,7 +6,7 @@ use crate::{
 };
 use arc_swap::ArcSwap;
 use futures::future::join_all;
-use std::{collections::HashMap, ops::Deref, sync::Arc};
+use std::{collections::HashMap, ops::Deref, option::Option, sync::Arc};
 use tokio::sync::Notify;
 
 /// Allows access to workers by task queue name
@@ -40,7 +40,7 @@ impl WorkerDispatcher {
             .workers
             .load()
             .get(&tq)
-            .map(|wo| wo.is_some())
+            .map(Option::is_some)
             .unwrap_or_default()
         {
             return Err(WorkerRegistrationError::WorkerAlreadyRegisteredForQueue(tq));
@@ -77,7 +77,7 @@ impl WorkerDispatcher {
             self.workers.rcu(|map| {
                 let mut map = HashMap::clone(map);
                 if maybe_worker.is_none() {
-                    maybe_worker = map.get_mut(task_queue).and_then(|o| o.take());
+                    maybe_worker = map.get_mut(task_queue).and_then(Option::take);
                 }
                 map
             });
@@ -149,7 +149,7 @@ impl Deref for WorkerRefCt {
     type Target = Worker;
 
     fn deref(&self) -> &Self::Target {
-        self.inner.as_ref().expect("Must exist").deref()
+        self.inner.as_deref().expect("Must exist")
     }
 }
 
@@ -161,7 +161,7 @@ impl Drop for WorkerRefCt {
             Some(arc) => {
                 // We wait until 2 rather than 1 because we ourselves still have an Arc
                 if Arc::strong_count(arc) == 2 {
-                    self.notify.notify_one()
+                    self.notify.notify_one();
                 }
             }
         };

package/sdk-core/src/worker/mod.rs

@@ -174,6 +174,8 @@ impl Worker {
                     ap,
                     sg.gw.clone(),
                     metrics.clone(),
+                    config.max_heartbeat_throttle_interval,
+                    config.default_heartbeat_throttle_interval,
                 )
             }),
             workflows_semaphore: Semaphore::new(config.max_outstanding_workflow_tasks),
@@ -253,7 +255,7 @@ impl Worker {
         if let Some(at_mgr) = self.at_task_mgr.as_ref() {
             let tt = details.task_token.clone();
             if let Err(e) = at_mgr.record_heartbeat(details) {
-                warn!(task_token = ?tt, details = ?e, "Activity heartbeat failed.")
+                warn!(task_token = ?tt, details = ?e, "Activity heartbeat failed.");
             }
         }
     }
@@ -327,20 +329,28 @@ impl Worker {
         completion: WfActivationCompletion,
     ) -> Result<(), CompleteWfError> {
         let wfstatus = completion.status;
-        let did_complete_wft = match wfstatus {
+        let report_outcome = match wfstatus {
             Some(wf_activation_completion::Status::Successful(success)) => {
                 self.wf_activation_success(&completion.run_id, success)
                     .await
             }
+
             Some(wf_activation_completion::Status::Failed(failure)) => {
-                self.wf_activation_failed(&completion.run_id, failure).await
+                self.wf_activation_failed(
+                    &completion.run_id,
+                    WorkflowTaskFailedCause::Unspecified,
+                    failure,
+                )
+                .await
+            }
+            None => {
+                return Err(CompleteWfError::MalformedWorkflowCompletion {
+                    reason: "Workflow completion had empty status field".to_owned(),
+                    completion: None,
+                })
             }
-            None => Err(CompleteWfError::MalformedWorkflowCompletion {
-                reason: "Workflow completion had empty status field".to_owned(),
-                completion: None,
-            }),
         }?;
-        self.after_workflow_activation(&completion.run_id, did_complete_wft);
+        self.after_workflow_activation(&completion.run_id, report_outcome);
         Ok(())
     }
 
@@ -354,7 +364,7 @@ impl Worker {
 
     /// Tell the worker a workflow task has completed, for tracking max outstanding WFTs
     pub(crate) fn return_workflow_task_permit(&self) {
-        self.workflows_semaphore.add_permits(1)
+        self.workflows_semaphore.add_permits(1);
     }
 
     pub(crate) fn request_wf_eviction(&self, run_id: &str, reason: impl Into<String>) {
@@ -486,6 +496,7 @@ impl Worker {
                         }),
                     )
                     .await?;
+                self.return_workflow_task_permit();
                 None
             }
             NewWfTaskOutcome::Evict(e) => {
@@ -506,7 +517,7 @@ impl Worker {
         &self,
         run_id: &str,
         success: workflow_completion::Success,
-    ) -> Result<bool, CompleteWfError> {
+    ) -> Result<WFTReportOutcome, CompleteWfError> {
         // Convert to wf commands
         let cmds = success
             .commands
@@ -550,7 +561,10 @@ impl Worker {
                         .await
                 })
                 .await?;
-                Ok(true)
+                Ok(WFTReportOutcome {
+                    reported_to_server: true,
+                    failed: false,
+                })
             }
             Ok(Some(ServerCommandsWithWorkflowInfo {
                 task_token,
@@ -560,9 +574,15 @@ impl Worker {
                 self.server_gateway
                     .respond_legacy_query(task_token, result)
                     .await?;
-                Ok(true)
+                Ok(WFTReportOutcome {
+                    reported_to_server: true,
+                    failed: false,
+                })
             }
-            Ok(None) => Ok(false),
+            Ok(None) => Ok(WFTReportOutcome {
+                reported_to_server: false,
+                failed: false,
+            }),
             Err(update_err) => {
                 // Automatically fail the workflow task in the event we couldn't update machines
                 let fail_cause = if matches!(&update_err.source, WFMachinesError::Nondeterminism(_))
@@ -571,30 +591,13 @@ impl Worker {
                 } else {
                     WorkflowTaskFailedCause::Unspecified
                 };
-
-                warn!(run_id, error=?update_err, "Failing workflow task");
-
-                if let Some(ref tt) = update_err.task_token {
-                    let wft_fail_str = format!("{:?}", update_err);
-                    self.handle_wft_reporting_errs(run_id, || async {
-                        self.server_gateway
-                            .fail_workflow_task(
-                                tt.clone(),
-                                fail_cause,
-                                Some(Failure::application_failure(wft_fail_str.clone(), false)),
-                            )
-                            .await
-                    })
-                    .await?;
-                    // We must evict the workflow since we've failed a WFT
-                    self.request_wf_eviction(
-                        run_id,
-                        format!("Workflow task failure: {}", wft_fail_str),
-                    );
-                    Ok(true)
-                } else {
-                    Ok(false)
-                }
+                let wft_fail_str = format!("{:?}", update_err);
+                self.wf_activation_failed(
+                    run_id,
+                    fail_cause,
+                    Failure::application_failure(wft_fail_str.clone(), false).into(),
+                )
+                .await
             }
         }
     }
@@ -605,35 +608,46 @@ impl Worker {
     async fn wf_activation_failed(
         &self,
         run_id: &str,
+        cause: WorkflowTaskFailedCause,
         failure: workflow_completion::Failure,
-    ) -> Result<bool, CompleteWfError> {
+    ) -> Result<WFTReportOutcome, CompleteWfError> {
         Ok(match self.wft_manager.failed_activation(run_id) {
             FailedActivationOutcome::Report(tt) => {
+                warn!(run_id, failure=?failure, "Failing workflow activation");
                 self.handle_wft_reporting_errs(run_id, || async {
                     self.server_gateway
-                        .fail_workflow_task(
-                            tt,
-                            WorkflowTaskFailedCause::Unspecified,
-                            failure.failure.map(Into::into),
-                        )
+                        .fail_workflow_task(tt, cause, failure.failure.map(Into::into))
                         .await
                 })
                 .await?;
-                true
+                WFTReportOutcome {
+                    reported_to_server: true,
+                    failed: true,
+                }
             }
             FailedActivationOutcome::ReportLegacyQueryFailure(task_token) => {
+                warn!(run_id, failure=?failure, "Failing legacy query request");
                 self.server_gateway
                     .respond_legacy_query(task_token, legacy_query_failure(failure))
                     .await?;
-                true
+                WFTReportOutcome {
+                    reported_to_server: true,
+                    failed: true,
+                }
             }
-            FailedActivationOutcome::NoReport => false,
+            FailedActivationOutcome::NoReport => WFTReportOutcome {
+                reported_to_server: false,
+                failed: true,
+            },
         })
     }
 
-    fn after_workflow_activation(&self, run_id: &str, did_complete_wft: bool) {
+    fn after_workflow_activation(&self, run_id: &str, report_outcome: WFTReportOutcome) {
         self.wft_manager.after_wft_report(run_id);
-        if did_complete_wft {
+        if report_outcome.reported_to_server || report_outcome.failed {
+            // If we failed the WFT but didn't report anything, we still want to release the WFT
+            // permit since the server will eventually time out the task and we've already evicted
+            // the run.
             self.return_workflow_task_permit();
         }
         self.wft_manager.on_activation_done(run_id);
@@ -715,6 +729,11 @@ impl WorkerConfig {
     }
 }
 
+struct WFTReportOutcome {
+    reported_to_server: bool,
+    failed: bool,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -731,7 +750,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_activities(5usize)
+            .max_outstanding_activities(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -749,7 +768,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_workflow_tasks(5usize)
+            .max_outstanding_workflow_tasks(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -767,7 +786,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_activities(5usize)
+            .max_outstanding_activities(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -785,7 +804,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_workflow_tasks(5usize)
+            .max_outstanding_workflow_tasks(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());

package/sdk-core/src/workflow/driven_workflow.rs

@@ -33,7 +33,7 @@ impl DrivenWorkflow {
     /// Start the workflow
     pub fn start(&mut self, attribs: WorkflowExecutionStartedEventAttributes) {
         debug!(run_id = %attribs.original_execution_run_id, "Driven WF start");
-        self.started_attrs = Some(attribs)
+        self.started_attrs = Some(attribs);
     }
 
     /// Enqueue a new job to be sent to the driven workflow
@@ -51,12 +51,12 @@ impl DrivenWorkflow {
 
     /// Signal the workflow
     pub fn signal(&mut self, signal: SignalWorkflow) {
-        self.send_job(wf_activation_job::Variant::SignalWorkflow(signal))
+        self.send_job(wf_activation_job::Variant::SignalWorkflow(signal));
     }
 
     /// Cancel the workflow
     pub fn cancel(&mut self, attribs: CancelWorkflow) {
-        self.send_job(wf_activation_job::Variant::CancelWorkflow(attribs))
+        self.send_job(wf_activation_job::Variant::CancelWorkflow(attribs));
     }
 }
 

package/sdk-core/src/workflow/history_update.rs

@@ -324,7 +324,7 @@ mod tests {
                 .take_next_wft_sequence(last_started_id)
                 .await
                 .unwrap();
-            for e in seq.iter() {
+            for e in &seq {
                 last_event_id += 1;
                 assert_eq!(e.event_id, last_event_id);
             }

package/sdk-core/src/workflow/mod.rs

@@ -62,7 +62,7 @@ impl WorkflowManager {
     }
 
     #[cfg(test)]
-    pub fn new_from_machines(workflow_machines: WorkflowMachines) -> Self {
+    pub const fn new_from_machines(workflow_machines: WorkflowMachines) -> Self {
         Self {
             machines: workflow_machines,
             command_sink: None,

package/sdk-core/src/workflow/workflow_tasks/cache_manager.rs

@@ -35,17 +35,13 @@ impl WorkflowCacheManager {
             // Blindly add a record into the cache, since it still has capacity.
             self.cache.put(run_id.to_owned(), ());
             None
-        } else if self.cache.cap() != 0 {
-            let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.to_owned());
-            let already_existed = self.cache.put(run_id.to_owned(), ()).is_some();
-            if !already_existed {
-                maybe_got_evicted
-            } else {
-                None
-            }
-        } else {
+        } else if self.cache.cap() == 0 {
             // Run id should be evicted right away as cache size is 0.
             Some(run_id.to_owned())
+        } else {
+            let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.clone());
+            let not_cached = self.cache.put(run_id.to_owned(), ()).is_none();
+            not_cached.then(|| maybe_got_evicted).flatten()
         };
 
         self.metrics.cache_size(self.cache.len() as u64);
@@ -75,7 +71,7 @@ mod tests {
         assert_matches!(wcm.insert("1"), None);
         assert_matches!(wcm.insert("2"), None);
         assert_matches!(wcm.insert("3"), Some(run_id) => {
-            assert_eq!(run_id, "1")
+            assert_eq!(run_id, "1");
         });
     }
 
@@ -88,7 +84,7 @@ mod tests {
         wcm.remove("1");
         assert_matches!(wcm.insert("2"), None);
         assert_matches!(wcm.insert("3"), Some(run_id) => {
-            assert_eq!(run_id, "2")
+            assert_eq!(run_id, "2");
         });
     }
 
@@ -110,7 +106,7 @@ mod tests {
         assert_matches!(wcm.insert("2"), None);
         wcm.touch("1");
         assert_matches!(wcm.insert("3"), Some(run_id) => {
-            assert_eq!(run_id, "2")
+            assert_eq!(run_id, "2");
         });
     }
 
@@ -123,7 +119,7 @@ mod tests {
         assert_matches!(wcm.insert("1"), None);
         assert_matches!(wcm.insert("2"), None);
         assert_matches!(wcm.insert("3"), Some(run_id) => {
-            assert_eq!(run_id, "1")
+            assert_eq!(run_id, "1");
         });
     }
 
@@ -133,10 +129,10 @@ mod tests {
             max_cached_workflows: 0,
         });
         assert_matches!(wcm.insert("1"), Some(run_id) => {
-            assert_eq!(run_id, "1")
+            assert_eq!(run_id, "1");
         });
         assert_matches!(wcm.insert("2"), Some(run_id) => {
-            assert_eq!(run_id, "2")
+            assert_eq!(run_id, "2");
         });
     }
 
@@ -144,10 +140,10 @@ mod tests {
     fn non_sticky_always_pending_eviction() {
         let mut wcm = WorkflowCacheManager::new_test(WorkflowCachingPolicy::NonSticky);
         assert_matches!(wcm.insert("1"), Some(run_id) => {
-            assert_eq!(run_id, "1")
+            assert_eq!(run_id, "1");
         });
         assert_matches!(wcm.insert("2"), Some(run_id) => {
-            assert_eq!(run_id, "2")
+            assert_eq!(run_id, "2");
         });
     }
 }

package/sdk-core/src/workflow/workflow_tasks/concurrency_manager.rs

@@ -184,11 +184,7 @@ impl WorkflowConcurrencyManager {
     pub fn delete_activation(&self, run_id: &str) -> Option<OutstandingActivation> {
         let mut writelock = self.runs.write();
         let machine_ref = writelock.get_mut(run_id);
-        if let Some(run) = machine_ref {
-            run.activation.take()
-        } else {
-            None
-        }
+        machine_ref.and_then(|run| run.activation.take())
     }
 
     pub fn exists(&self, run_id: &str) -> bool {
@@ -270,14 +266,14 @@ impl WorkflowConcurrencyManager {
     /// Remove the workflow with the provided run id from management
     pub fn evict(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
         let val = self.runs.write().remove(run_id);
-        val.map(|v| v.buffered_resp).flatten()
+        val.and_then(|v| v.buffered_resp)
     }
 
     /// Clear and return any buffered polling response for this run ID
     pub fn take_buffered_poll(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
         let mut writelock = self.runs.write();
         let val = writelock.get_mut(run_id);
-        val.map(|v| v.buffered_resp.take()).flatten()
+        val.and_then(|v| v.buffered_resp.take())
     }
 
     pub fn outstanding_wft(&self) -> usize {
@@ -311,6 +307,6 @@ mod tests {
             )
             .await;
         // Should whine that the machines have nothing to do (history empty)
-        assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. })
+        assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. });
     }
 }