@temporalio/core-bridge 1.11.7 → 1.11.8

package/sdk-core/core/src/worker/nexus.rs

@@ -0,0 +1,410 @@
+use crate::{
+    abstractions::UsedMeteredSemPermit,
+    pollers::{BoxedNexusPoller, NexusPollItem, new_nexus_task_poller},
+    telemetry::{
+        metrics,
+        metrics::{FailureReason, MetricsContext},
+    },
+    worker::client::WorkerClient,
+};
+use anyhow::anyhow;
+use futures_util::{
+    Stream, StreamExt, stream,
+    stream::{BoxStream, PollNext},
+};
+use std::{
+    collections::HashMap,
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    },
+    time::{Duration, Instant, SystemTime},
+};
+use temporal_sdk_core_api::{
+    errors::{CompleteNexusError, PollError},
+    worker::NexusSlotKind,
+};
+use temporal_sdk_core_protos::{
+    TaskToken,
+    coresdk::{
+        NexusSlotInfo,
+        nexus::{
+            CancelNexusTask, NexusTask, NexusTaskCancelReason, nexus_task, nexus_task_completion,
+        },
+    },
+    temporal::api::nexus::v1::{request::Variant, response, start_operation_response},
+};
+use tokio::{
+    join,
+    sync::{Mutex, Notify, mpsc::UnboundedSender},
+    task::JoinHandle,
+};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_util::sync::CancellationToken;
+
+static REQUEST_TIMEOUT_HEADER: &str = "Request-Timeout";
+
+/// Centralizes all state related to received nexus tasks
+pub(super) struct NexusManager {
+    task_stream: Mutex<BoxStream<'static, Result<NexusTask, PollError>>>,
+    /// Token to notify when poll returned a shutdown error
+    poll_returned_shutdown_token: CancellationToken,
+    /// Outstanding nexus tasks that have been issued to lang but not yet completed
+    outstanding_task_map: OutstandingTaskMap,
+    /// Notified every time a task in the map is completed
+    task_completed_notify: Arc<Notify>,
+
+    ever_polled: AtomicBool,
+    metrics: MetricsContext,
+}
+
+impl NexusManager {
+    pub(super) fn new(
+        poller: BoxedNexusPoller,
+        metrics: MetricsContext,
+        graceful_shutdown: Option<Duration>,
+        shutdown_initiated_token: CancellationToken,
+    ) -> Self {
+        let source_stream =
+            new_nexus_task_poller(poller, metrics.clone(), shutdown_initiated_token);
+        let (cancels_tx, cancels_rx) = tokio::sync::mpsc::unbounded_channel();
+        let task_stream_input = stream::select_with_strategy(
+            UnboundedReceiverStream::new(cancels_rx).map(TaskStreamInput::from),
+            source_stream
+                .map(TaskStreamInput::from)
+                .chain(stream::once(async move { TaskStreamInput::SourceComplete })),
+            |_: &mut ()| PollNext::Left,
+        );
+        let task_completed_notify = Arc::new(Notify::new());
+        let task_stream = NexusTaskStream::new(
+            task_stream_input,
+            cancels_tx,
+            task_completed_notify.clone(),
+            graceful_shutdown,
+            metrics.clone(),
+        );
+        let outstanding_task_map = task_stream.outstanding_task_map.clone();
+        Self {
+            task_stream: Mutex::new(task_stream.into_stream().boxed()),
+            poll_returned_shutdown_token: CancellationToken::new(),
+            outstanding_task_map,
+            task_completed_notify,
+            ever_polled: AtomicBool::new(false),
+            metrics,
+        }
+    }
+
+    /// Block until then next nexus task is received from server
+    pub(super) async fn next_nexus_task(&self) -> Result<NexusTask, PollError> {
+        self.ever_polled.store(true, Ordering::Relaxed);
+        let mut sl = self.task_stream.lock().await;
+        let r = sl.next().await.unwrap_or_else(|| Err(PollError::ShutDown));
+        // This can't happen in the or_else closure because ShutDown is typically returned by the
+        // stream directly, before it terminates.
+        if let Err(PollError::ShutDown) = &r {
+            self.poll_returned_shutdown_token.cancel();
+        }
+        r
+    }
+
+    pub(super) async fn complete_task(
+        &self,
+        tt: TaskToken,
+        status: nexus_task_completion::Status,
+        client: &dyn WorkerClient,
+    ) -> Result<(), CompleteNexusError> {
+        let removed = self.outstanding_task_map.lock().remove(&tt);
+        if let Some(task_info) = removed {
+            self.metrics
+                .nexus_task_execution_latency(task_info.start_time.elapsed());
+            task_info.timeout_task.inspect(|jh| jh.abort());
+            let (did_send, maybe_net_err) = match status {
+                nexus_task_completion::Status::Completed(c) => {
+                    // Server doesn't provide obvious errors for this validation, so it's done
+                    // here to make life easier for lang implementors.
+                    match &c.variant {
+                        Some(response::Variant::StartOperation(so)) => {
+                            if let Some(start_operation_response::Variant::OperationError(oe)) =
+                                so.variant.as_ref()
+                            {
+                                self.metrics
+                                    .with_new_attrs([metrics::failure_reason(
+                                        FailureReason::NexusOperation(oe.operation_state.clone()),
+                                    )])
+                                    .nexus_task_execution_failed();
+                            };
+                            if task_info.request_kind != RequestKind::Start {
+                                return Err(CompleteNexusError::MalformedNexusCompletion {
+                                    reason: "Nexus response was StartOperation but request was not"
+                                        .to_string(),
+                                });
+                            }
+                        }
+                        Some(response::Variant::CancelOperation(_)) => {
+                            if task_info.request_kind != RequestKind::Cancel {
+                                return Err(CompleteNexusError::MalformedNexusCompletion {
+                                    reason:
+                                        "Nexus response was CancelOperation but request was not"
+                                            .to_string(),
+                                });
+                            }
+                        }
+                        None => {
+                            return Err(CompleteNexusError::MalformedNexusCompletion {
+                                reason: "Nexus completion must contain a status variant "
+                                    .to_string(),
+                            });
+                        }
+                    }
+                    (true, client.complete_nexus_task(tt, c).await.err())
+                }
+                nexus_task_completion::Status::AckCancel(_) => {
+                    self.metrics
+                        .with_new_attrs([metrics::failure_reason(FailureReason::Timeout)])
+                        .nexus_task_execution_failed();
+                    (false, None)
+                }
+                nexus_task_completion::Status::Error(e) => {
+                    self.metrics
+                        .with_new_attrs([metrics::failure_reason(
+                            FailureReason::NexusHandlerError(e.error_type.clone()),
+                        )])
+                        .nexus_task_execution_failed();
+                    (true, client.fail_nexus_task(tt, e).await.err())
+                }
+            };
+
+            self.task_completed_notify.notify_waiters();
+
+            if let Some(e) = maybe_net_err {
+                if e.code() == tonic::Code::NotFound {
+                    warn!(details=?e, "Nexus task not found on completion. This \
+                    may happen if the operation has already been cancelled but completed anyway.");
+                } else {
+                    warn!(error=?e, "Network error while completing Nexus task");
+                }
+            } else if did_send {
+                // Record e2e latency if we sent replied to server without an RPC error
+                if let Some(elapsed) = task_info.scheduled_time.and_then(|t| t.elapsed().ok()) {
+                    self.metrics.nexus_task_e2e_latency(elapsed);
+                }
+            }
+        } else {
+            warn!(
+                "Attempted to complete nexus task {} but we were not tracking it",
+                &tt
+            );
+        }
+        Ok(())
+    }
+
+    pub(super) async fn shutdown(&self) {
+        if !self.ever_polled.load(Ordering::Relaxed) {
+            return;
+        }
+        self.poll_returned_shutdown_token.cancelled().await;
+    }
+}
+
+struct NexusTaskStream<S> {
+    source_stream: S,
+    outstanding_task_map: OutstandingTaskMap,
+    cancels_tx: UnboundedSender<CancelNexusTask>,
+    task_completed_notify: Arc<Notify>,
+    grace_period: Option<Duration>,
+    metrics: MetricsContext,
+}
+
+impl<S> NexusTaskStream<S>
+where
+    S: Stream<Item = TaskStreamInput>,
+{
+    fn new(
+        source: S,
+        cancels_tx: UnboundedSender<CancelNexusTask>,
+        task_completed_notify: Arc<Notify>,
+        grace_period: Option<Duration>,
+        metrics: MetricsContext,
+    ) -> Self {
+        Self {
+            source_stream: source,
+            outstanding_task_map: Arc::new(Default::default()),
+            cancels_tx,
+            task_completed_notify,
+            grace_period,
+            metrics,
+        }
+    }
+
+    fn into_stream(self) -> impl Stream<Item = Result<NexusTask, PollError>> {
+        let outstanding_task_clone = self.outstanding_task_map.clone();
+        let source_done = CancellationToken::new();
+        let source_done_clone = source_done.clone();
+        let cancels_tx_clone = self.cancels_tx.clone();
+        self.source_stream
+            .filter_map(move |t| {
+                let res = match t {
+                    TaskStreamInput::Poll(Ok(t)) => {
+                        if let Some(dur) = t.resp.sched_to_start() {
+                            self.metrics.nexus_task_sched_to_start_latency(dur);
+                        };
+
+                        let tt = TaskToken(t.resp.task_token.clone());
+                        let mut timeout_task = None;
+                        if let Some(timeout_str) = t
+                            .resp
+                            .request
+                            .as_ref()
+                            .and_then(|r| r.header.get(REQUEST_TIMEOUT_HEADER))
+                        {
+                            if let Ok(timeout_dur) = parse_request_timeout(timeout_str) {
+                                let tt_clone = tt.clone();
+                                let cancels_tx = self.cancels_tx.clone();
+                                timeout_task = Some(tokio::task::spawn(async move {
+                                    tokio::time::sleep(timeout_dur).await;
+                                    debug!(
+                                        task_token=%tt_clone,
+                                        "Timing out nexus task due to elapsed local timeout timer"
+                                    );
+                                    let _ = cancels_tx.send(CancelNexusTask {
+                                        task_token: tt_clone.0,
+                                        reason: NexusTaskCancelReason::TimedOut.into(),
+                                    });
+                                }));
+                            } else {
+                                // This could auto-respond and fail the nexus task, but given that
+                                // the server is going to try to parse this as well, and all we're
+                                // doing with this parsing is notifying the handler of a local
+                                // timeout, it seems reasonable to rely on server to handle this.
+                                warn!(
+                                    "Failed to parse nexus timeout header value '{}'",
+                                    timeout_str
+                                );
+                            }
+                        }
+
+                        let (service, operation, request_kind) = t
+                            .resp
+                            .request
+                            .as_ref()
+                            .and_then(|r| r.variant.as_ref())
+                            .map(|v| match v {
+                                Variant::StartOperation(s) => (
+                                    s.service.to_owned(),
+                                    s.operation.to_owned(),
+                                    RequestKind::Start,
+                                ),
+                                Variant::CancelOperation(c) => (
+                                    c.service.to_owned(),
+                                    c.operation.to_owned(),
+                                    RequestKind::Cancel,
+                                ),
+                            })
+                            .unwrap_or_default();
+                        self.outstanding_task_map.lock().insert(
+                            tt,
+                            NexusInFlightTask {
+                                request_kind,
+                                timeout_task,
+                                scheduled_time: t
+                                    .resp
+                                    .request
+                                    .as_ref()
+                                    .and_then(|r| r.scheduled_time)
+                                    .and_then(|t| t.try_into().ok()),
+                                start_time: Instant::now(),
+                                _permit: t.permit.into_used(NexusSlotInfo { service, operation }),
+                            },
+                        );
+                        Some(Ok(NexusTask {
+                            variant: Some(nexus_task::Variant::Task(t.resp)),
+                        }))
+                    }
+                    TaskStreamInput::Cancel(c) => Some(Ok(NexusTask {
+                        variant: Some(nexus_task::Variant::CancelTask(c)),
+                    })),
+                    TaskStreamInput::SourceComplete => {
+                        source_done.cancel();
+                        None
+                    }
+                    TaskStreamInput::Poll(Err(e)) => Some(Err(PollError::TonicError(e))),
+                };
+                async move { res }
+            })
+            .take_until(async move {
+                source_done_clone.cancelled().await;
+                let (grace_killer, stop_grace) = futures_util::future::abortable(async {
+                    if let Some(gp) = self.grace_period {
+                        tokio::time::sleep(gp).await;
+                        for (tt, _) in outstanding_task_clone.lock().iter() {
+                            let _ = cancels_tx_clone.send(CancelNexusTask {
+                                task_token: tt.0.clone(),
+                                reason: NexusTaskCancelReason::WorkerShutdown.into(),
+                            });
+                        }
+                    }
+                });
+                join!(
+                    async {
+                        while !outstanding_task_clone.lock().is_empty() {
+                            self.task_completed_notify.notified().await;
+                        }
+                        // If we were waiting for the grace period but everything already finished,
+                        // we don't need to keep waiting.
+                        stop_grace.abort();
+                    },
+                    grace_killer
+                )
+            })
+            .chain(stream::once(async move { Err(PollError::ShutDown) }))
+    }
+}
+
+type OutstandingTaskMap = Arc<parking_lot::Mutex<HashMap<TaskToken, NexusInFlightTask>>>;
+
+struct NexusInFlightTask {
+    request_kind: RequestKind,
+    timeout_task: Option<JoinHandle<()>>,
+    scheduled_time: Option<SystemTime>,
+    start_time: Instant,
+    _permit: UsedMeteredSemPermit<NexusSlotKind>,
+}
+
+#[derive(Eq, PartialEq, Copy, Clone, Default)]
+enum RequestKind {
+    #[default]
+    Start,
+    Cancel,
+}
+
+#[derive(derive_more::From)]
+enum TaskStreamInput {
+    Poll(NexusPollItem),
+    Cancel(CancelNexusTask),
+    SourceComplete,
+}
+
+fn parse_request_timeout(timeout: &str) -> Result<Duration, anyhow::Error> {
+    let timeout = timeout.trim();
+    let (value, unit) = timeout.split_at(
+        timeout
+            .find(|c: char| !c.is_ascii_digit() && c != '.')
+            .unwrap_or(timeout.len()),
+    );
+
+    match unit {
+        "m" => value
+            .parse::<f64>()
+            .map(|v| Duration::from_secs_f64(60.0 * v))
+            .map_err(Into::into),
+        "s" => value
+            .parse::<f64>()
+            .map(Duration::from_secs_f64)
+            .map_err(Into::into),
+        "ms" => value
+            .parse::<u64>()
+            .map(Duration::from_millis)
+            .map_err(Into::into),
+        _ => Err(anyhow!("Invalid timeout format")),
+    }
+}

package/sdk-core/core/src/worker/tuner/resource_based.rs

@@ -3,17 +3,17 @@ use parking_lot::Mutex;
 use std::{
     marker::PhantomData,
     sync::{
-        atomic::{AtomicU64, AtomicUsize, Ordering},
         Arc, OnceLock,
+        atomic::{AtomicU64, AtomicUsize, Ordering},
     },
     time::{Duration, Instant},
 };
 use temporal_sdk_core_api::{
     telemetry::metrics::{CoreMeter, GaugeF64, MetricAttributes, TemporalMeter},
     worker::{
-        ActivitySlotKind, LocalActivitySlotKind, SlotInfo, SlotInfoTrait, SlotKind, SlotKindType,
-        SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext, SlotSupplier,
-        SlotSupplierPermit, WorkerTuner, WorkflowSlotKind,
+        ActivitySlotKind, LocalActivitySlotKind, NexusSlotKind, SlotInfo, SlotInfoTrait, SlotKind,
+        SlotKindType, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
+        SlotSupplier, SlotSupplierPermit, WorkerTuner, WorkflowSlotKind,
     },
 };
 use tokio::{sync::watch, task::JoinHandle};
@@ -30,6 +30,7 @@ pub struct ResourceBasedTuner<MI> {
     wf_opts: Option<ResourceSlotOptions>,
     act_opts: Option<ResourceSlotOptions>,
     la_opts: Option<ResourceSlotOptions>,
+    nexus_opts: Option<ResourceSlotOptions>,
 }
 
 impl ResourceBasedTuner<RealSysInfo> {
@@ -59,6 +60,7 @@ impl<MI> ResourceBasedTuner<MI> {
             wf_opts: None,
             act_opts: None,
             la_opts: None,
+            nexus_opts: None,
         }
     }
 
@@ -79,6 +81,12 @@ impl<MI> ResourceBasedTuner<MI> {
         self.la_opts = Some(opts);
         self
     }
+
+    /// Set nexus slot options
+    pub fn with_nexus_slots_options(&mut self, opts: ResourceSlotOptions) -> &mut Self {
+        self.nexus_opts = Some(opts);
+        self
+    }
 }
 
 const DEFAULT_WF_SLOT_OPTS: ResourceSlotOptions = ResourceSlotOptions {
@@ -91,6 +99,13 @@ const DEFAULT_ACT_SLOT_OPTS: ResourceSlotOptions = ResourceSlotOptions {
     max_slots: 10_000,
     ramp_throttle: Duration::from_millis(50),
 };
+const DEFAULT_NEXUS_SLOT_OPTS: ResourceSlotOptions = ResourceSlotOptions {
+    min_slots: 1,
+    max_slots: 10_000,
+    // No ramp is chosen under the assumption that nexus tasks are unlikely to use many resources
+    // and would prefer lowest latency over protection against oversubscription.
+    ramp_throttle: Duration::from_millis(0),
+};
 
 /// Options for a specific slot type
 #[derive(Debug, Clone, Copy, derive_more::Constructor)]
@@ -375,6 +390,13 @@ impl<MI: SystemResourceInfo + Sync + Send + 'static> WorkerTuner for ResourceBas
         self.slots.as_kind(o)
     }
 
+    fn nexus_task_slot_supplier(
+        &self,
+    ) -> Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync> {
+        let o = self.nexus_opts.unwrap_or(DEFAULT_NEXUS_SLOT_OPTS);
+        self.slots.as_kind(o)
+    }
+
     fn attach_metrics(&self, metrics: TemporalMeter) {
         self.slots.attach_metrics(metrics);
     }
@@ -524,8 +546,8 @@ mod tests {
     use super::*;
     use crate::{abstractions::MeteredPermitDealer, telemetry::metrics::MetricsContext};
     use std::sync::{
-        atomic::{AtomicU64, Ordering},
         Arc,
+        atomic::{AtomicU64, Ordering},
     };
     use temporal_sdk_core_api::worker::WorkflowSlotKind;
 

package/sdk-core/core/src/worker/tuner.rs

@@ -11,8 +11,8 @@ use std::sync::{Arc, OnceLock};
 use temporal_sdk_core_api::{
     telemetry::metrics::TemporalMeter,
     worker::{
-        ActivitySlotKind, LocalActivitySlotKind, SlotKind, SlotSupplier, WorkerConfig, WorkerTuner,
-        WorkflowSlotKind,
+        ActivitySlotKind, LocalActivitySlotKind, NexusSlotKind, SlotKind, SlotSupplier,
+        WorkerConfig, WorkerTuner, WorkflowSlotKind,
     },
 };
 
@@ -21,6 +21,7 @@ pub struct TunerHolder {
     wft_supplier: Arc<dyn SlotSupplier<SlotKind = WorkflowSlotKind> + Send + Sync>,
     act_supplier: Arc<dyn SlotSupplier<SlotKind = ActivitySlotKind> + Send + Sync>,
     la_supplier: Arc<dyn SlotSupplier<SlotKind = LocalActivitySlotKind> + Send + Sync>,
+    nexus_supplier: Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync>,
     metrics: OnceLock<TemporalMeter>,
 }
 
@@ -39,6 +40,9 @@ pub struct TunerHolderOptions {
     /// Options for local activity slots
     #[builder(default, setter(strip_option))]
     pub local_activity_slot_options: Option<SlotSupplierOptions<LocalActivitySlotKind>>,
+    /// Options for nexus slots
+    #[builder(default, setter(strip_option))]
+    pub nexus_slot_options: Option<SlotSupplierOptions<NexusSlotKind>>,
     /// Options that will apply to all resource based slot suppliers. Must be set if any slot
     /// options are [SlotSupplierOptions::ResourceBased]
     #[builder(default, setter(strip_option))]
@@ -165,6 +169,7 @@ pub struct TunerBuilder {
         Option<Arc<dyn SlotSupplier<SlotKind = ActivitySlotKind> + Send + Sync>>,
     local_activity_slot_supplier:
         Option<Arc<dyn SlotSupplier<SlotKind = LocalActivitySlotKind> + Send + Sync>>,
+    nexus_slot_supplier: Option<Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync>>,
 }
 
 impl TunerBuilder {
@@ -179,6 +184,9 @@ impl TunerBuilder {
         if let Some(m) = cfg.max_outstanding_local_activities {
             builder.local_activity_slot_supplier(Arc::new(FixedSizeSlotSupplier::new(m)));
         }
+        if let Some(m) = cfg.max_outstanding_nexus_tasks {
+            builder.nexus_slot_supplier(Arc::new(FixedSizeSlotSupplier::new(m)));
+        }
         builder
     }
 
@@ -209,6 +217,15 @@ impl TunerBuilder {
         self
     }
 
+    /// Set a nexus slot supplier
+    pub fn nexus_slot_supplier(
+        &mut self,
+        supplier: Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync>,
+    ) -> &mut Self {
+        self.nexus_slot_supplier = Some(supplier);
+        self
+    }
+
     /// Build a [WorkerTuner] from the configured slot suppliers
     pub fn build(&mut self) -> TunerHolder {
         TunerHolder {
@@ -224,6 +241,10 @@ impl TunerBuilder {
                 .local_activity_slot_supplier
                 .clone()
                 .unwrap_or_else(|| Arc::new(FixedSizeSlotSupplier::new(100))),
+            nexus_supplier: self
+                .nexus_slot_supplier
+                .clone()
+                .unwrap_or_else(|| Arc::new(FixedSizeSlotSupplier::new(100))),
             metrics: OnceLock::new(),
         }
     }
@@ -248,6 +269,12 @@ impl WorkerTuner for TunerHolder {
         self.la_supplier.clone()
     }
 
+    fn nexus_task_slot_supplier(
+        &self,
+    ) -> Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync> {
+        self.nexus_supplier.clone()
+    }
+
     fn attach_metrics(&self, m: TemporalMeter) {
         let _ = self.metrics.set(m);
     }

package/sdk-core/core/src/worker/workflow/driven_workflow.rs

@@ -1,6 +1,6 @@
 use crate::{
     telemetry::VecDisplayer,
-    worker::workflow::{OutgoingJob, WFCommand, WorkflowStartedInfo},
+    worker::workflow::{OutgoingJob, WFCommand, WFCommandVariant, WorkflowStartedInfo},
 };
 use prost_types::Timestamp;
 use std::{
@@ -8,7 +8,7 @@ use std::{
     sync::mpsc::{self, Receiver, Sender},
 };
 use temporal_sdk_core_protos::{
-    coresdk::workflow_activation::{start_workflow_from_attribs, WorkflowActivationJob},
+    coresdk::workflow_activation::{WorkflowActivationJob, start_workflow_from_attribs},
     temporal::api::{common::v1::Payload, history::v1::WorkflowExecutionStartedEventAttributes},
     utilities::TryIntoOrNone,
 };
@@ -86,7 +86,12 @@ impl DrivenWorkflow {
     /// from a buffer that the language side sinks into when it calls [crate::Core::complete_task]
     pub(super) fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand> {
         let in_cmds = self.incoming_commands.try_recv();
-        let in_cmds = in_cmds.unwrap_or_else(|_| vec![WFCommand::NoCommandsFromLang]);
+        let in_cmds = in_cmds.unwrap_or_else(|_| {
+            vec![WFCommand {
+                variant: WFCommandVariant::NoCommandsFromLang,
+                metadata: None,
+            }]
+        });
         debug!(in_cmds = %in_cmds.display(), "wf bridge iteration fetch");
         in_cmds
     }

package/sdk-core/core/src/worker/workflow/history_update.rs

@@ -5,7 +5,7 @@ use crate::{
         workflow::{CacheMissFetchReq, PermittedWFT, PreparedWFT},
     },
 };
-use futures_util::{future::BoxFuture, FutureExt, Stream, TryFutureExt};
+use futures_util::{FutureExt, Stream, TryFutureExt, future::BoxFuture};
 use itertools::Itertools;
 use std::{
     collections::VecDeque,
@@ -20,8 +20,7 @@ use std::{
 use temporal_sdk_core_protos::temporal::api::{
     enums::v1::EventType,
     history::v1::{
-        history_event, history_event::Attributes, History, HistoryEvent,
-        WorkflowTaskCompletedEventAttributes,
+        History, HistoryEvent, WorkflowTaskCompletedEventAttributes, history_event::Attributes,
     },
 };
 use tracing::Instrument;
@@ -651,9 +650,7 @@ impl HistoryUpdate {
             .iter()
             .skip_while(|e| e.event_id < from_id)
             .find_map(|e| match &e.attributes {
-                Some(history_event::Attributes::WorkflowTaskCompletedEventAttributes(ref a)) => {
-                    Some(a)
-                }
+                Some(Attributes::WorkflowTaskCompletedEventAttributes(a)) => Some(a),
                 _ => None,
             })
     }
@@ -794,7 +791,7 @@ mod tests {
     use super::*;
     use crate::{
         replay::{HistoryInfo, TestHistoryBuilder},
-        test_help::{canned_histories, hist_to_poll_resp, mock_sdk_cfg, MockPollCfg, ResponseType},
+        test_help::{MockPollCfg, ResponseType, canned_histories, hist_to_poll_resp, mock_sdk_cfg},
         worker::client::mocks::mock_workflow_client,
     };
     use futures_util::{StreamExt, TryStreamExt};
@@ -802,11 +799,11 @@ mod tests {
     use temporal_client::WorkflowOptions;
     use temporal_sdk::WfContext;
     use temporal_sdk_core_protos::{
+        DEFAULT_WORKFLOW_TYPE,
         temporal::api::{
             common::v1::WorkflowExecution, enums::v1::WorkflowTaskFailedCause,
             workflowservice::v1::GetWorkflowExecutionHistoryResponse,
         },
-        DEFAULT_WORKFLOW_TYPE,
     };
 
     impl From<HistoryInfo> for HistoryUpdate {