@temporalio/core-bridge 0.13.0 → 0.16.3

package/sdk-core/src/worker/activities.rs

@@ -10,7 +10,6 @@ use activity_heartbeat_manager::ActivityHeartbeatManager;
 use dashmap::DashMap;
 use std::{
     convert::TryInto,
-    ops::Div,
     sync::Arc,
     time::{Duration, Instant},
 };
@@ -80,6 +79,9 @@ pub(crate) struct WorkerActivityTasks {
     activities_semaphore: Semaphore,
 
     metrics: MetricsContext,
+
+    max_heartbeat_throttle_interval: Duration,
+    default_heartbeat_throttle_interval: Duration,
 }
 
 impl WorkerActivityTasks {
@@ -88,6 +90,8 @@ impl WorkerActivityTasks {
         poller: BoxedActPoller,
         sg: Arc<impl ServerGatewayApis + Send + Sync + 'static + ?Sized>,
         metrics: MetricsContext,
+        max_heartbeat_throttle_interval: Duration,
+        default_heartbeat_throttle_interval: Duration,
     ) -> Self {
         Self {
             heartbeat_manager: ActivityHeartbeatManager::new(sg),
@@ -95,12 +99,13 @@ impl WorkerActivityTasks {
             poller,
             activities_semaphore: Semaphore::new(max_activity_tasks),
             metrics,
+            max_heartbeat_throttle_interval,
+            default_heartbeat_throttle_interval,
         }
     }
 
     pub(crate) fn notify_shutdown(&self) {
         self.poller.notify_shutdown();
-        self.heartbeat_manager.notify_shutdown();
     }
 
     pub(crate) async fn shutdown(self) {
@@ -171,17 +176,21 @@ impl WorkerActivityTasks {
         status: activity_result::Status,
         gateway: &(dyn ServerGatewayApis + Send + Sync),
     ) -> Result<(), CompleteActivityError> {
-        if let Some(act_info) = self.outstanding_activity_tasks.get(&task_token) {
+        if let Some((_, act_info)) = self.outstanding_activity_tasks.remove(&task_token) {
             let act_metrics = self.metrics.with_new_attrs([
                 activity_type(act_info.activity_type.clone()),
                 workflow_type(act_info.workflow_type.clone()),
             ]);
             act_metrics.act_execution_latency(act_info.start_time.elapsed());
+            self.activities_semaphore.add_permits(1);
+            self.heartbeat_manager.evict(task_token.clone());
+            let known_not_found = act_info.known_not_found;
+            drop(act_info); // TODO: Get rid of dashmap. If we hold ref across await, bad stuff.
 
             // No need to report activities which we already know the server doesn't care about
-            let should_remove = if !act_info.known_not_found {
-                drop(act_info); // TODO: Get rid of dashmap. If we hold ref across await, bad stuff.
+            if !known_not_found {
                 let maybe_net_err = match status {
+                    activity_result::Status::WillCompleteAsync(_) => None,
                     activity_result::Status::Completed(ar::Success { result }) => gateway
                         .complete_activity_task(task_token.clone(), result.map(Into::into))
                         .await
@@ -194,19 +203,17 @@ impl WorkerActivityTasks {
                             .err()
                     }
                     activity_result::Status::Cancelled(ar::Cancellation { failure }) => {
-                        let details = match failure {
-                            Some(Failure {
-                                failure_info:
-                                    Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo {
-                                        details,
-                                    })),
-                                ..
-                            }) => details,
-                            _ => {
-                                warn!(task_token = ? task_token,
-                                    "Expected activity cancelled status with CanceledFailureInfo");
-                                None
-                            }
+                        let details = if let Some(Failure {
+                            failure_info:
+                                Some(FailureInfo::CanceledFailureInfo(CanceledFailureInfo { details })),
+                            ..
+                        }) = failure
+                        {
+                            details
+                        } else {
+                            warn!(task_token = ? task_token,
+                                "Expected activity cancelled status with CanceledFailureInfo");
+                            None
                         };
                         gateway
                             .cancel_activity_task(task_token.clone(), details.map(Into::into))
@@ -214,37 +221,24 @@ impl WorkerActivityTasks {
                             .err()
                     }
                 };
-                match maybe_net_err {
-                    Some(e) if e.code() == tonic::Code::NotFound => {
+
+                if let Some(e) = maybe_net_err {
+                    if e.code() == tonic::Code::NotFound {
                         warn!(task_token = ?task_token, details = ?e, "Activity not found on \
                         completion. This may happen if the activity has already been cancelled but \
                         completed anyway.");
-                        true
-                    }
-                    Some(err) => return Err(err.into()),
-                    None => true,
-                }
-            } else {
-                true
+                    } else {
+                        return Err(e.into());
+                    };
+                };
             };
-
-            if should_remove
-                && self
-                    .outstanding_activity_tasks
-                    .remove(&task_token)
-                    .is_some()
-            {
-                self.activities_semaphore.add_permits(1);
-                self.heartbeat_manager.evict(task_token);
-            }
-            Ok(())
         } else {
             warn!(
                 "Attempted to complete activity task {} but we were not tracking it",
                 &task_token
             );
-            Ok(())
         }
+        Ok(())
     }
 
     /// Attempt to record an activity heartbeat
@@ -253,22 +247,30 @@ impl WorkerActivityTasks {
         details: ActivityHeartbeat,
     ) -> Result<(), ActivityHeartbeatError> {
         // TODO: Propagate these back as cancels. Silent fails is too nonobvious
-        let t: Duration = self
+        let heartbeat_timeout: Duration = self
             .outstanding_activity_tasks
             .get(&TaskToken(details.task_token.clone()))
             .ok_or(ActivityHeartbeatError::UnknownActivity)?
             .heartbeat_timeout
             .clone()
-            .ok_or(ActivityHeartbeatError::HeartbeatTimeoutNotSet)?
+            // We treat None as 0 (even though heartbeat_timeout is never set to None by the server)
+            .unwrap_or_default()
             .try_into()
+            // This technically should never happen since prost duration should be directly mappable
+            // to std::time::Duration.
             .or(Err(ActivityHeartbeatError::InvalidHeartbeatTimeout))?;
+
         // There is a bug in the server that translates non-set heartbeat timeouts into 0 duration.
         // That's why we treat 0 the same way as None, otherwise we wouldn't know which aggregation
         // delay to use, and using 0 is not a good idea as SDK would hammer the server too hard.
-        if t.as_millis() == 0 {
-            return Err(ActivityHeartbeatError::HeartbeatTimeoutNotSet);
-        }
-        self.heartbeat_manager.record(details, t.div(2))
+        let throttle_interval = if heartbeat_timeout.as_millis() == 0 {
+            self.default_heartbeat_throttle_interval
+        } else {
+            heartbeat_timeout.mul_f64(0.8)
+        };
+        let throttle_interval =
+            std::cmp::min(throttle_interval, self.max_heartbeat_throttle_interval);
+        self.heartbeat_manager.record(details, throttle_interval)
     }
 
     async fn next_pending_cancel_task(&self) -> Result<Option<ActivityTask>, PollActivityError> {

package/sdk-core/src/worker/config.rs

@@ -51,6 +51,17 @@ pub struct WorkerConfig {
     /// and moved to the non-sticky queue where it may be picked up by any worker.
     #[builder(default = "Duration::from_secs(10)")]
     pub sticky_queue_schedule_to_start_timeout: Duration,
+
+    /// Longest interval for throttling activity heartbeats
+    #[builder(default = "Duration::from_secs(60)")]
+    pub max_heartbeat_throttle_interval: Duration,
+
+    /// Default interval for throttling activity heartbeats in case
+    /// `ActivityOptions.heartbeat_timeout` is unset.
+    /// When the timeout *is* set in the `ActivityOptions`, throttling is set to
+    /// `heartbeat_timeout * 0.8`.
+    #[builder(default = "Duration::from_secs(30)")]
+    pub default_heartbeat_throttle_interval: Duration,
 }
 
 impl WorkerConfigBuilder {

package/sdk-core/src/worker/dispatcher.rs

@@ -6,7 +6,7 @@ use crate::{
 };
 use arc_swap::ArcSwap;
 use futures::future::join_all;
-use std::{collections::HashMap, ops::Deref, sync::Arc};
+use std::{collections::HashMap, ops::Deref, option::Option, sync::Arc};
 use tokio::sync::Notify;
 
 /// Allows access to workers by task queue name
@@ -40,7 +40,7 @@ impl WorkerDispatcher {
             .workers
             .load()
             .get(&tq)
-            .map(|wo| wo.is_some())
+            .map(Option::is_some)
             .unwrap_or_default()
         {
             return Err(WorkerRegistrationError::WorkerAlreadyRegisteredForQueue(tq));
@@ -77,7 +77,7 @@ impl WorkerDispatcher {
             self.workers.rcu(|map| {
                 let mut map = HashMap::clone(map);
                 if maybe_worker.is_none() {
-                    maybe_worker = map.get_mut(task_queue).and_then(|o| o.take());
+                    maybe_worker = map.get_mut(task_queue).and_then(Option::take);
                 }
                 map
             });
@@ -149,7 +149,7 @@ impl Deref for WorkerRefCt {
     type Target = Worker;
 
     fn deref(&self) -> &Self::Target {
-        self.inner.as_ref().expect("Must exist").deref()
+        self.inner.as_deref().expect("Must exist")
     }
 }
 
@@ -161,7 +161,7 @@ impl Drop for WorkerRefCt {
             Some(arc) => {
                 // We wait until 2 rather than 1 because we ourselves still have an Arc
                 if Arc::strong_count(arc) == 2 {
-                    self.notify.notify_one()
+                    self.notify.notify_one();
                 }
             }
         };

package/sdk-core/src/worker/mod.rs

@@ -6,7 +6,7 @@ pub use crate::worker::config::{WorkerConfig, WorkerConfigBuilder};
 pub(crate) use dispatcher::WorkerDispatcher;
 
 use crate::{
-    errors::{CompleteWfError, WorkflowUpdateError},
+    errors::CompleteWfError,
     machines::{EmptyWorkflowCommandErr, WFMachinesError},
     pollers::{
         new_activity_task_buffer, new_workflow_task_buffer, BoxedActPoller, BoxedWFPoller,
@@ -98,6 +98,7 @@ impl Worker {
         let mut wf_task_poll_buffer = new_workflow_task_buffer(
             sg.gw.clone(),
             config.task_queue.clone(),
+            false,
             max_nonsticky_polls,
             max_nonsticky_polls * 2,
         );
@@ -107,6 +108,7 @@ impl Worker {
             let mut sp = new_workflow_task_buffer(
                 sg.gw.clone(),
                 sqn.clone(),
+                true,
                 max_sticky_polls,
                 max_sticky_polls * 2,
             );
@@ -172,6 +174,8 @@ impl Worker {
                     ap,
                     sg.gw.clone(),
                     metrics.clone(),
+                    config.max_heartbeat_throttle_interval,
+                    config.default_heartbeat_throttle_interval,
                 )
             }),
             workflows_semaphore: Semaphore::new(config.max_outstanding_workflow_tasks),
@@ -219,6 +223,11 @@ impl Worker {
         self.wft_manager.outstanding_wft()
     }
 
+    #[cfg(test)]
+    pub(crate) fn available_wft_permits(&self) -> usize {
+        self.workflows_semaphore.available_permits()
+    }
+
     /// Wait until not at the outstanding activity limit, and then poll this worker's task queue for
     /// new activities.
     ///
@@ -246,7 +255,7 @@ impl Worker {
         if let Some(at_mgr) = self.at_task_mgr.as_ref() {
             let tt = details.task_token.clone();
             if let Err(e) = at_mgr.record_heartbeat(details) {
-                warn!(task_token = ?tt, details = ?e, "Activity heartbeat failed.")
+                warn!(task_token = ?tt, details = ?e, "Activity heartbeat failed.");
             }
         }
     }
@@ -275,7 +284,7 @@ impl Worker {
             // We must first check if there are pending workflow activations for workflows that are
             // currently replaying or otherwise need immediate jobs, and issue those before
             // bothering the server.
-            if let Some(pa) = self.wft_manager.next_pending_activation()? {
+            if let Some(pa) = self.wft_manager.next_pending_activation() {
                 debug!(activation=%pa, "Sending pending activation to lang");
                 return Ok(pa);
             }
@@ -284,7 +293,7 @@ impl Worker {
             // activations, since there may be an eviction etc for whatever run is popped here.
             if let Some(buff_wft) = self.wft_manager.next_buffered_poll() {
                 match self.apply_server_work(buff_wft).await? {
-                    NewWfTaskOutcome::IssueActivation(a) => return Ok(a),
+                    Some(a) => return Ok(a),
                     _ => continue,
                 }
             }
@@ -304,14 +313,8 @@ impl Worker {
 
             if let Some(work) = selected_f {
                 self.metrics.wf_tq_poll_ok();
-                match self.apply_server_work(work).await? {
-                    NewWfTaskOutcome::IssueActivation(a) => return Ok(a),
-                    NewWfTaskOutcome::TaskBuffered => {
-                        // If the task was buffered, it's not actually outstanding, so we can
-                        // immediately return a permit.
-                        self.return_workflow_task_permit();
-                    }
-                    _ => {}
+                if let Some(a) = self.apply_server_work(work).await? {
+                    return Ok(a);
                 }
             }
 
@@ -326,7 +329,7 @@ impl Worker {
         completion: WfActivationCompletion,
     ) -> Result<(), CompleteWfError> {
         let wfstatus = completion.status;
-        let r = match wfstatus {
+        let did_complete_wft = match wfstatus {
             Some(wf_activation_completion::Status::Successful(success)) => {
                 self.wf_activation_success(&completion.run_id, success)
                     .await
@@ -338,11 +341,9 @@ impl Worker {
                 reason: "Workflow completion had empty status field".to_owned(),
                 completion: None,
             }),
-        };
-        self.after_wft_report(&completion.run_id)?;
-        self.wft_manager.on_activation_done(&completion.run_id);
-        self.maybe_notify_wtfs_drained();
-        r
+        }?;
+        self.after_workflow_activation(&completion.run_id, did_complete_wft);
+        Ok(())
     }
 
     fn maybe_notify_wtfs_drained(&self) {
@@ -355,11 +356,11 @@ impl Worker {
 
     /// Tell the worker a workflow task has completed, for tracking max outstanding WFTs
     pub(crate) fn return_workflow_task_permit(&self) {
-        self.workflows_semaphore.add_permits(1)
+        self.workflows_semaphore.add_permits(1);
     }
 
-    pub(crate) fn request_wf_eviction(&self, run_id: &str) {
-        self.wft_manager.request_eviction(run_id);
+    pub(crate) fn request_wf_eviction(&self, run_id: &str, reason: impl Into<String>) {
+        self.wft_manager.request_eviction(run_id, reason);
     }
 
     /// Resolves with WFT poll response or `PollWfError::ShutDown` if WFTs have been drained
@@ -443,18 +444,24 @@ impl Worker {
     async fn apply_server_work(
         &self,
         work: ValidPollWFTQResponse,
-    ) -> Result<NewWfTaskOutcome, PollWfError> {
+    ) -> Result<Option<WfActivation>, PollWfError> {
         let we = work.workflow_execution.clone();
         let tt = work.task_token.clone();
         let res = self
             .wft_manager
             .apply_new_poll_resp(work, &self.server_gateway)
-            .await?;
-        match &res {
+            .await;
+        Ok(match res {
             NewWfTaskOutcome::IssueActivation(a) => {
                 debug!(activation=%a, "Sending activation to lang");
+                Some(a)
+            }
+            NewWfTaskOutcome::TaskBuffered => {
+                // If the task was buffered, it's not actually outstanding, so we can
+                // immediately return a permit.
+                self.return_workflow_task_permit();
+                None
             }
-            NewWfTaskOutcome::TaskBuffered => {}
             NewWfTaskOutcome::Autocomplete => {
                 debug!(workflow_execution=?we,
                        "No work for lang to perform after polling server. Sending autocomplete.");
@@ -464,6 +471,7 @@ impl Worker {
                     status: Some(workflow_completion::Success::from_variants(vec![]).into()),
                 })
                 .await?;
+                None
             }
             NewWfTaskOutcome::CacheMiss => {
                 debug!(workflow_execution=?we, "Unable to process workflow task with partial \
@@ -480,17 +488,27 @@ impl Worker {
                         }),
                     )
                     .await?;
+                None
             }
-        };
-        Ok(res)
+            NewWfTaskOutcome::Evict(e) => {
+                warn!(error=?e, run_id=%we.run_id, "Error while applying poll response to workflow");
+                self.request_wf_eviction(
+                    &we.run_id,
+                    format!("Error while applying poll response to workflow: {:?}", e),
+                );
+                None
+            }
+        })
     }
 
-    /// Handle a successful workflow completion
+    /// Handle a successful workflow activation
+    ///
+    /// Returns true if we actually reported WFT completion to server (success or failure)
     async fn wf_activation_success(
         &self,
         run_id: &str,
         success: workflow_completion::Success,
-    ) -> Result<(), CompleteWfError> {
+    ) -> Result<bool, CompleteWfError> {
         // Convert to wf commands
         let cmds = success
             .commands
@@ -534,6 +552,7 @@ impl Worker {
                         .await
                 })
                 .await?;
+                Ok(true)
             }
             Ok(Some(ServerCommandsWithWorkflowInfo {
                 task_token,
@@ -543,8 +562,9 @@ impl Worker {
                 self.server_gateway
                     .respond_legacy_query(task_token, result)
                     .await?;
+                Ok(true)
             }
-            Ok(None) => {}
+            Ok(None) => Ok(false),
             Err(update_err) => {
                 // Automatically fail the workflow task in the event we couldn't update machines
                 let fail_cause = if matches!(&update_err.source, WFMachinesError::Nondeterminism(_))
@@ -554,35 +574,42 @@ impl Worker {
                     WorkflowTaskFailedCause::Unspecified
                 };
 
+                warn!(run_id, error=?update_err, "Failing workflow task");
+
                 if let Some(ref tt) = update_err.task_token {
+                    let wft_fail_str = format!("{:?}", update_err);
                     self.handle_wft_reporting_errs(run_id, || async {
                         self.server_gateway
                             .fail_workflow_task(
                                 tt.clone(),
                                 fail_cause,
-                                Some(Failure::application_failure(
-                                    format!("{:?}", update_err),
-                                    false,
-                                )),
+                                Some(Failure::application_failure(wft_fail_str.clone(), false)),
                             )
                             .await
                     })
                     .await?;
+                    // We must evict the workflow since we've failed a WFT
+                    self.request_wf_eviction(
+                        run_id,
+                        format!("Workflow task failure: {}", wft_fail_str),
+                    );
+                    Ok(true)
+                } else {
+                    Ok(false)
                 }
-                return Err(update_err.into());
             }
         }
-
-        Ok(())
     }
 
     /// Handle a failed workflow completion
+    ///
+    /// Returns true if we actually reported WFT completion to server
     async fn wf_activation_failed(
         &self,
         run_id: &str,
         failure: workflow_completion::Failure,
-    ) -> Result<(), CompleteWfError> {
-        match self.wft_manager.failed_activation(run_id) {
+    ) -> Result<bool, CompleteWfError> {
+        Ok(match self.wft_manager.failed_activation(run_id) {
             FailedActivationOutcome::Report(tt) => {
                 self.handle_wft_reporting_errs(run_id, || async {
                     self.server_gateway
@@ -594,23 +621,25 @@ impl Worker {
                         .await
                 })
                 .await?;
+                true
             }
             FailedActivationOutcome::ReportLegacyQueryFailure(task_token) => {
                 self.server_gateway
                     .respond_legacy_query(task_token, legacy_query_failure(failure))
                     .await?;
+                true
             }
-            _ => {}
-        }
-
-        Ok(())
+            FailedActivationOutcome::NoReport => false,
+        })
     }
 
-    fn after_wft_report(&self, run_id: &str) -> Result<(), WorkflowUpdateError> {
-        if self.wft_manager.after_wft_report(run_id)? {
+    fn after_workflow_activation(&self, run_id: &str, did_complete_wft: bool) {
+        self.wft_manager.after_wft_report(run_id);
+        if did_complete_wft {
             self.return_workflow_task_permit();
-        };
-        Ok(())
+        }
+        self.wft_manager.on_activation_done(run_id);
+        self.maybe_notify_wtfs_drained();
     }
 
     /// Handle server errors from either completing or failing a workflow task. Returns any errors
@@ -630,11 +659,12 @@ impl Worker {
                     // Silence unhandled command errors since the lang SDK cannot do anything about
                     // them besides poll again, which it will do anyway.
                     tonic::Code::InvalidArgument if err.message() == "UnhandledCommand" => {
-                        warn!("Unhandled command response when completing: {}", err);
+                        warn!(error = %err, "Unhandled command response when completing");
+                        should_evict = true;
                         Ok(())
                     }
                     tonic::Code::NotFound => {
-                        warn!("Task not found when completing: {}", err);
+                        warn!(error = %err, "Task not found when completing");
                         should_evict = true;
                         Ok(())
                     }
@@ -644,7 +674,7 @@ impl Worker {
             _ => Ok(()),
         };
         if should_evict {
-            self.wft_manager.request_eviction(run_id);
+            self.request_wf_eviction(run_id, "Error reporting WFT to server");
         }
         res.map_err(Into::into)
     }
@@ -703,7 +733,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_activities(5usize)
+            .max_outstanding_activities(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -716,12 +746,12 @@ mod tests {
         let mut mock_gateway = MockServerGatewayApis::new();
         mock_gateway
             .expect_poll_workflow_task()
-            .returning(|_| Ok(PollWorkflowTaskQueueResponse::default()));
+            .returning(|_, _| Ok(PollWorkflowTaskQueueResponse::default()));
         let gwref = GatewayRef::new(Arc::new(mock_gateway), fake_sg_opts());
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_workflow_tasks(5usize)
+            .max_outstanding_workflow_tasks(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -739,7 +769,7 @@ mod tests {
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_activities(5usize)
+            .max_outstanding_activities(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());
@@ -752,12 +782,12 @@ mod tests {
         let mut mock_gateway = MockServerGatewayApis::new();
         mock_gateway
             .expect_poll_workflow_task()
-            .returning(|_| Err(tonic::Status::internal("ahhh")));
+            .returning(|_, _| Err(tonic::Status::internal("ahhh")));
         let gwref = GatewayRef::new(Arc::new(mock_gateway), fake_sg_opts());
 
         let cfg = WorkerConfigBuilder::default()
             .task_queue("whatever")
-            .max_outstanding_workflow_tasks(5usize)
+            .max_outstanding_workflow_tasks(5_usize)
             .build()
             .unwrap();
         let worker = Worker::new(cfg, None, Arc::new(gwref), Default::default());

package/sdk-core/src/workflow/driven_workflow.rs

@@ -33,7 +33,7 @@ impl DrivenWorkflow {
     /// Start the workflow
     pub fn start(&mut self, attribs: WorkflowExecutionStartedEventAttributes) {
         debug!(run_id = %attribs.original_execution_run_id, "Driven WF start");
-        self.started_attrs = Some(attribs)
+        self.started_attrs = Some(attribs);
     }
 
     /// Enqueue a new job to be sent to the driven workflow
@@ -51,12 +51,12 @@ impl DrivenWorkflow {
 
     /// Signal the workflow
     pub fn signal(&mut self, signal: SignalWorkflow) {
-        self.send_job(wf_activation_job::Variant::SignalWorkflow(signal))
+        self.send_job(wf_activation_job::Variant::SignalWorkflow(signal));
     }
 
     /// Cancel the workflow
     pub fn cancel(&mut self, attribs: CancelWorkflow) {
-        self.send_job(wf_activation_job::Variant::CancelWorkflow(attribs))
+        self.send_job(wf_activation_job::Variant::CancelWorkflow(attribs));
     }
 }
 

package/sdk-core/src/workflow/history_update.rs

@@ -324,7 +324,7 @@ mod tests {
                 .take_next_wft_sequence(last_started_id)
                 .await
                 .unwrap();
-            for e in seq.iter() {
+            for e in &seq {
                 last_event_id += 1;
                 assert_eq!(e.event_id, last_event_id);
             }

package/sdk-core/src/workflow/mod.rs

@@ -62,7 +62,7 @@ impl WorkflowManager {
     }
 
     #[cfg(test)]
-    pub fn new_from_machines(workflow_machines: WorkflowMachines) -> Self {
+    pub const fn new_from_machines(workflow_machines: WorkflowMachines) -> Self {
         Self {
             machines: workflow_machines,
             command_sink: None,
@@ -277,6 +277,7 @@ pub mod managed_wf {
             // Send an eviction to ensure wf exits if it has not finished (ex: feeding partial hist)
             let _ = self.activation_tx.send(create_evict_activation(
                 "not actually important".to_string(),
+                "force shutdown".to_string(),
             ));
             self.future_handle.take().unwrap().await.unwrap()
         }