@temporalio/core-bridge 1.13.0 → 1.13.2

package/sdk-core/core/src/worker/client.rs

@@ -1,18 +1,18 @@
 //! Worker-specific client needs
 
 pub(crate) mod mocks;
-use crate::abstractions::dbg_panic;
-use crate::protosext::legacy_query_failure;
-use crate::worker::heartbeat::HeartbeatFn;
-use parking_lot::RwLock;
-use std::sync::OnceLock;
-use std::{sync::Arc, time::Duration};
+use crate::{
+    abstractions::dbg_panic, protosext::legacy_query_failure, worker::heartbeat::HeartbeatFn,
+};
+use std::{
+    sync::{Arc, OnceLock},
+    time::Duration,
+};
 use temporal_client::{
     Client, IsWorkerTaskLongPoll, Namespace, NamespacedClient, NoRetryOnMatching, RetryClient,
-    SlotManager, WorkflowService,
+    SharedReplaceableClient, SlotManager, WorkflowService,
 };
 use temporal_sdk_core_api::worker::WorkerVersioningStrategy;
-use temporal_sdk_core_protos::temporal::api::worker::v1::WorkerHeartbeat;
 use temporal_sdk_core_protos::{
     TaskToken,
     coresdk::{workflow_commands::QueryResult, workflow_completion},
@@ -32,6 +32,7 @@ use temporal_sdk_core_protos::{
         query::v1::WorkflowQueryResult,
         sdk::v1::WorkflowTaskCompletedMetadata,
         taskqueue::v1::{StickyExecutionAttributes, TaskQueue, TaskQueueMetadata},
+        worker::v1::WorkerHeartbeat,
         workflowservice::v1::{get_system_info_response::Capabilities, *},
     },
 };
@@ -46,7 +47,7 @@ pub enum LegacyQueryResult {
 
 /// Contains everything a worker needs to interact with the server
 pub(crate) struct WorkerClientBag {
-    replaceable_client: RwLock<RetryClient<Client>>,
+    client: RetryClient<SharedReplaceableClient<Client>>,
     namespace: String,
     identity: String,
     worker_versioning_strategy: WorkerVersioningStrategy,
@@ -55,14 +56,14 @@ pub(crate) struct WorkerClientBag {
 
 impl WorkerClientBag {
     pub(crate) fn new(
-        client: RetryClient<Client>,
+        client: RetryClient<SharedReplaceableClient<Client>>,
         namespace: String,
         identity: String,
         worker_versioning_strategy: WorkerVersioningStrategy,
         heartbeat_data: Option<Arc<OnceLock<HeartbeatFn>>>,
     ) -> Self {
         Self {
-            replaceable_client: RwLock::new(client),
+            client,
             namespace,
             identity,
             worker_versioning_strategy,
@@ -70,10 +71,6 @@ impl WorkerClientBag {
         }
     }
 
-    fn cloned_client(&self) -> RetryClient<Client> {
-        self.replaceable_client.read().clone()
-    }
-
     fn default_capabilities(&self) -> Capabilities {
         self.capabilities().unwrap_or_default()
     }
@@ -144,7 +141,7 @@ impl WorkerClientBag {
 
 /// This trait contains everything workers need to interact with Temporal, and hence provides a
 /// minimal mocking surface. Delegates to [WorkflowClientTrait] so see that for details.
-#[cfg_attr(test, mockall::automock)]
+#[cfg_attr(any(feature = "test-utilities", test), mockall::automock)]
 #[async_trait::async_trait]
 pub trait WorkerClient: Sync + Send {
     /// Poll workflow tasks
@@ -236,7 +233,7 @@ pub trait WorkerClient: Sync + Send {
     ) -> Result<RecordWorkerHeartbeatResponse>;
 
     /// Replace the underlying client
-    fn replace_client(&self, new_client: RetryClient<Client>);
+    fn replace_client(&self, new_client: Client);
     /// Return server capabilities
     fn capabilities(&self) -> Option<Capabilities>;
     /// Return workers using this client
@@ -312,7 +309,8 @@ impl WorkerClient for WorkerClientBag {
         }
 
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .poll_workflow_task_queue(request)
             .await?
             .into_inner())
@@ -349,7 +347,8 @@ impl WorkerClient for WorkerClientBag {
         }
 
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .poll_activity_task_queue(request)
             .await?
             .into_inner())
@@ -382,7 +381,8 @@ impl WorkerClient for WorkerClientBag {
         }
 
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .poll_nexus_task_queue(request)
             .await?
             .into_inner())
@@ -432,8 +432,9 @@ impl WorkerClient for WorkerClientBag {
             deployment_options: self.deployment_options(),
         };
         Ok(self
-            .cloned_client()
-            .respond_workflow_task_completed(request)
+            .client
+            .clone()
+            .respond_workflow_task_completed(request.into_request())
             .await?
             .into_inner())
     }
@@ -444,7 +445,8 @@ impl WorkerClient for WorkerClientBag {
         result: Option<Payloads>,
     ) -> Result<RespondActivityTaskCompletedResponse> {
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .respond_activity_task_completed(
                 #[allow(deprecated)] // want to list all fields explicitly
                 RespondActivityTaskCompletedRequest {
@@ -456,7 +458,8 @@ impl WorkerClient for WorkerClientBag {
                     // Will never be set, deprecated.
                     deployment: None,
                     deployment_options: self.deployment_options(),
-                },
+                }
+                .into_request(),
             )
             .await?
             .into_inner())
@@ -468,13 +471,17 @@ impl WorkerClient for WorkerClientBag {
         response: nexus::v1::Response,
     ) -> Result<RespondNexusTaskCompletedResponse> {
         Ok(self
-            .cloned_client()
-            .respond_nexus_task_completed(RespondNexusTaskCompletedRequest {
-                namespace: self.namespace.clone(),
-                identity: self.identity.clone(),
-                task_token: task_token.0,
-                response: Some(response),
-            })
+            .client
+            .clone()
+            .respond_nexus_task_completed(
+                RespondNexusTaskCompletedRequest {
+                    namespace: self.namespace.clone(),
+                    identity: self.identity.clone(),
+                    task_token: task_token.0,
+                    response: Some(response),
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
@@ -485,13 +492,17 @@ impl WorkerClient for WorkerClientBag {
         details: Option<Payloads>,
     ) -> Result<RecordActivityTaskHeartbeatResponse> {
         Ok(self
-            .cloned_client()
-            .record_activity_task_heartbeat(RecordActivityTaskHeartbeatRequest {
-                task_token: task_token.0,
-                details,
-                identity: self.identity.clone(),
-                namespace: self.namespace.clone(),
-            })
+            .client
+            .clone()
+            .record_activity_task_heartbeat(
+                RecordActivityTaskHeartbeatRequest {
+                    task_token: task_token.0,
+                    details,
+                    identity: self.identity.clone(),
+                    namespace: self.namespace.clone(),
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
@@ -502,7 +513,8 @@ impl WorkerClient for WorkerClientBag {
         details: Option<Payloads>,
     ) -> Result<RespondActivityTaskCanceledResponse> {
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .respond_activity_task_canceled(
                 #[allow(deprecated)] // want to list all fields explicitly
                 RespondActivityTaskCanceledRequest {
@@ -514,7 +526,8 @@ impl WorkerClient for WorkerClientBag {
                     // Will never be set, deprecated.
                     deployment: None,
                     deployment_options: self.deployment_options(),
-                },
+                }
+                .into_request(),
             )
             .await?
             .into_inner())
@@ -526,7 +539,8 @@ impl WorkerClient for WorkerClientBag {
         failure: Option<Failure>,
     ) -> Result<RespondActivityTaskFailedResponse> {
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .respond_activity_task_failed(
                 #[allow(deprecated)] // want to list all fields explicitly
                 RespondActivityTaskFailedRequest {
@@ -540,7 +554,8 @@ impl WorkerClient for WorkerClientBag {
                     // Will never be set, deprecated.
                     deployment: None,
                     deployment_options: self.deployment_options(),
-                },
+                }
+                .into_request(),
             )
             .await?
             .into_inner())
@@ -567,8 +582,9 @@ impl WorkerClient for WorkerClientBag {
             deployment_options: self.deployment_options(),
         };
         Ok(self
-            .cloned_client()
-            .respond_workflow_task_failed(request)
+            .client
+            .clone()
+            .respond_workflow_task_failed(request.into_request())
             .await?
             .into_inner())
     }
@@ -579,13 +595,17 @@ impl WorkerClient for WorkerClientBag {
         error: nexus::v1::HandlerError,
     ) -> Result<RespondNexusTaskFailedResponse> {
         Ok(self
-            .cloned_client()
-            .respond_nexus_task_failed(RespondNexusTaskFailedRequest {
-                namespace: self.namespace.clone(),
-                identity: self.identity.clone(),
-                task_token: task_token.0,
-                error: Some(error),
-            })
+            .client
+            .clone()
+            .respond_nexus_task_failed(
+                RespondNexusTaskFailedRequest {
+                    namespace: self.namespace.clone(),
+                    identity: self.identity.clone(),
+                    task_token: task_token.0,
+                    error: Some(error),
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
@@ -597,16 +617,20 @@ impl WorkerClient for WorkerClientBag {
         page_token: Vec<u8>,
     ) -> Result<GetWorkflowExecutionHistoryResponse> {
         Ok(self
-            .cloned_client()
-            .get_workflow_execution_history(GetWorkflowExecutionHistoryRequest {
-                namespace: self.namespace.clone(),
-                execution: Some(WorkflowExecution {
-                    workflow_id,
-                    run_id: run_id.unwrap_or_default(),
-                }),
-                next_page_token: page_token,
-                ..Default::default()
-            })
+            .client
+            .clone()
+            .get_workflow_execution_history(
+                GetWorkflowExecutionHistoryRequest {
+                    namespace: self.namespace.clone(),
+                    execution: Some(WorkflowExecution {
+                        workflow_id,
+                        run_id: run_id.unwrap_or_default(),
+                    }),
+                    next_page_token: page_token,
+                    ..Default::default()
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
@@ -629,25 +653,32 @@ impl WorkerClient for WorkerClientBag {
         let (_, completed_type, query_result, error_message) = query_result.into_components();
 
         Ok(self
-            .cloned_client()
-            .respond_query_task_completed(RespondQueryTaskCompletedRequest {
-                task_token: task_token.into(),
-                completed_type: completed_type as i32,
-                query_result,
-                error_message,
-                namespace: self.namespace.clone(),
-                failure,
-                cause: cause.into(),
-            })
+            .client
+            .clone()
+            .respond_query_task_completed(
+                RespondQueryTaskCompletedRequest {
+                    task_token: task_token.into(),
+                    completed_type: completed_type as i32,
+                    query_result,
+                    error_message,
+                    namespace: self.namespace.clone(),
+                    failure,
+                    cause: cause.into(),
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
 
     async fn describe_namespace(&self) -> Result<DescribeNamespaceResponse> {
         Ok(self
-            .cloned_client()
+            .client
+            .clone()
             .describe_namespace(
-                Namespace::Name(self.namespace.clone()).into_describe_namespace_request(),
+                Namespace::Name(self.namespace.clone())
+                    .into_describe_namespace_request()
+                    .into_request(),
             )
             .await?
             .into_inner())
@@ -663,15 +694,14 @@ impl WorkerClient for WorkerClientBag {
         };
 
         Ok(
-            WorkflowService::shutdown_worker(&mut self.cloned_client(), request)
+            WorkflowService::shutdown_worker(&mut self.client.clone(), request.into_request())
                 .await?
                 .into_inner(),
         )
     }
 
-    fn replace_client(&self, new_client: RetryClient<Client>) {
-        let mut replaceable_client = self.replaceable_client.write();
-        *replaceable_client = new_client;
+    fn replace_client(&self, new_client: Client) {
+        self.client.get_client().replace_client(new_client);
     }
 
     async fn record_worker_heartbeat(
@@ -679,24 +709,31 @@ impl WorkerClient for WorkerClientBag {
         heartbeat: WorkerHeartbeat,
     ) -> Result<RecordWorkerHeartbeatResponse> {
         Ok(self
-            .cloned_client()
-            .record_worker_heartbeat(RecordWorkerHeartbeatRequest {
-                namespace: self.namespace.clone(),
-                identity: self.identity.clone(),
-                worker_heartbeat: vec![heartbeat],
-            })
+            .client
+            .clone()
+            .record_worker_heartbeat(
+                RecordWorkerHeartbeatRequest {
+                    namespace: self.namespace.clone(),
+                    identity: self.identity.clone(),
+                    worker_heartbeat: vec![heartbeat],
+                }
+                .into_request(),
+            )
             .await?
             .into_inner())
     }
 
     fn capabilities(&self) -> Option<Capabilities> {
-        let client = self.replaceable_client.read();
-        client.get_client().inner().capabilities().cloned()
+        self.client
+            .get_client()
+            .inner_cow()
+            .inner()
+            .capabilities()
+            .cloned()
     }
 
     fn workers(&self) -> Arc<SlotManager> {
-        let client = self.replaceable_client.read();
-        client.get_client().inner().workers()
+        self.client.get_client().inner_cow().inner().workers()
     }
 
     fn is_mock(&self) -> bool {
@@ -704,8 +741,8 @@ impl WorkerClient for WorkerClientBag {
     }
 
     fn sdk_name_and_version(&self) -> (String, String) {
-        let lock = self.replaceable_client.read();
-        let opts = lock.get_client().inner().options();
+        let inner = self.client.get_client().inner_cow();
+        let opts = inner.options();
         (opts.client_name.clone(), opts.client_version.clone())
     }
 
@@ -715,12 +752,12 @@ impl WorkerClient for WorkerClientBag {
 }
 
 impl NamespacedClient for WorkerClientBag {
-    fn namespace(&self) -> &str {
-        &self.namespace
+    fn namespace(&self) -> String {
+        self.namespace.clone()
     }
 
-    fn get_identity(&self) -> &str {
-        &self.identity
+    fn identity(&self) -> String {
+        self.identity.clone()
     }
 }
 

package/sdk-core/core/src/worker/heartbeat.rs

@@ -1,15 +1,14 @@
-use crate::WorkerClient;
-use crate::abstractions::dbg_panic;
+use crate::{WorkerClient, abstractions::dbg_panic};
 use gethostname::gethostname;
 use parking_lot::Mutex;
 use prost_types::Duration as PbDuration;
-use std::sync::{Arc, OnceLock};
-use std::time::{Duration, SystemTime};
+use std::{
+    sync::{Arc, OnceLock},
+    time::{Duration, SystemTime},
+};
 use temporal_sdk_core_api::worker::WorkerConfig;
 use temporal_sdk_core_protos::temporal::api::worker::v1::{WorkerHeartbeat, WorkerHostInfo};
-use tokio::sync::Notify;
-use tokio::task::JoinHandle;
-use tokio::time::MissedTickBehavior;
+use tokio::{sync::Notify, task::JoinHandle, time::MissedTickBehavior};
 use uuid::Uuid;
 
 pub(crate) type HeartbeatFn = Box<dyn Fn() -> Option<WorkerHeartbeat> + Send + Sync>;
@@ -169,12 +168,12 @@ impl WorkerHeartbeatData {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_help::WorkerExt;
-    use crate::test_help::test_worker_cfg;
-    use crate::worker;
-    use crate::worker::client::mocks::mock_worker_client;
-    use std::sync::Arc;
-    use std::time::Duration;
+    use crate::{
+        test_help::{WorkerExt, test_worker_cfg},
+        worker,
+        worker::client::mocks::mock_worker_client,
+    };
+    use std::{sync::Arc, time::Duration};
     use temporal_sdk_core_api::worker::PollerBehavior;
     use temporal_sdk_core_protos::temporal::api::workflowservice::v1::RecordWorkerHeartbeatResponse;
 

package/sdk-core/core/src/worker/mod.rs

@@ -18,15 +18,17 @@ pub(crate) use activities::{
     NewLocalAct,
 };
 pub(crate) use wft_poller::WFTPollerShared;
-pub(crate) use workflow::LEGACY_QUERY_ID;
 
-use crate::worker::heartbeat::{HeartbeatFn, WorkerHeartbeatManager};
+#[allow(unreachable_pub)] // re-exported in test_help::integ_helpers
+pub use workflow::LEGACY_QUERY_ID;
+
 use crate::{
     ActivityHeartbeat, CompleteActivityError, PollError, WorkerTrait,
     abstractions::{MeteredPermitDealer, PermitDealerContextData, dbg_panic},
     errors::CompleteWfError,
-    pollers::{BoxedActPoller, BoxedNexusPoller},
+    pollers::{ActivityTaskOptions, BoxedActPoller, BoxedNexusPoller, LongPollBuffer},
     protosext::validate_activity_completion,
+    sealed::AnyClient,
     telemetry::{
         TelemetryInstance,
         metrics::{
@@ -37,31 +39,28 @@ use crate::{
     worker::{
         activities::{LACompleteAction, LocalActivityManager, NextPendingLAAction},
         client::WorkerClient,
+        heartbeat::{HeartbeatFn, WorkerHeartbeatManager},
         nexus::NexusManager,
         workflow::{
-            LAReqSink, LocalResolution, WorkflowBasics, Workflows, wft_poller::make_wft_poller,
+            LAReqSink, LocalResolution, WorkflowBasics, Workflows, wft_poller,
+            wft_poller::make_wft_poller,
         },
     },
 };
-use crate::{
-    pollers::{ActivityTaskOptions, LongPollBuffer},
-    worker::workflow::wft_poller,
-};
 use activities::WorkerActivityTasks;
 use futures_util::{StreamExt, stream};
 use parking_lot::Mutex;
 use slot_provider::SlotProvider;
-use std::sync::OnceLock;
 use std::{
     convert::TryInto,
     future,
     sync::{
-        Arc,
+        Arc, OnceLock,
         atomic::{AtomicBool, Ordering},
     },
     time::Duration,
 };
-use temporal_client::{ConfiguredClient, TemporalServiceClientWithMetrics, WorkerKey};
+use temporal_client::WorkerKey;
 use temporal_sdk_core_api::{
     errors::{CompleteNexusError, WorkerValidationError},
     worker::PollerBehavior,
@@ -84,7 +83,8 @@ use temporal_sdk_core_protos::{
 use tokio::sync::{mpsc::unbounded_channel, watch};
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tokio_util::sync::CancellationToken;
-#[cfg(test)]
+
+#[cfg(any(feature = "test-utilities", test))]
 use {
     crate::{
         pollers::{BoxedPoller, MockPermittedPollBuffer},
@@ -114,7 +114,7 @@ pub struct Worker {
     shutdown_token: CancellationToken,
     /// Will be called at the end of each activation completion
     #[allow(clippy::type_complexity)] // Sorry clippy, there's no simple way to re-use here.
-    post_activate_hook: Option<Box<dyn Fn(&Self, PostActivateHookData) + Send + Sync>>,
+    post_activate_hook: Option<Box<dyn Fn(&Self, PostActivateHookData<'_>) + Send + Sync>>,
     /// Set when non-local activities are complete and should stop being polled
     non_local_activities_complete: Arc<AtomicBool>,
     /// Set when local activities are complete and should stop being polled
@@ -289,9 +289,12 @@ impl Worker {
         )
     }
 
-    /// Replace client and return a new client. For eager workflow purposes, this new client will
-    /// now apply to future eager start requests and the older client will not.
-    pub fn replace_client(&self, new_client: ConfiguredClient<TemporalServiceClientWithMetrics>) {
+    /// Replace client. For eager workflow purposes, this new client will now apply to future
+    /// eager start requests and the older client will not.
+    pub fn replace_client<CT>(&self, new_client: CT)
+    where
+        CT: Into<AnyClient>,
+    {
         // Unregister worker from current client, register in new client at the end
         let mut worker_key = self.worker_key.lock();
         let slot_provider = (*worker_key).and_then(|k| self.client.workers().unregister(k));
@@ -396,7 +399,7 @@ impl Worker {
                         shutdown_token.child_token(),
                         Some(move |np| act_metrics.record_num_pollers(np)),
                         ActivityTaskOptions {
-                            max_worker_acts_per_second: config.max_task_queue_activities_per_second,
+                            max_worker_acts_per_second: config.max_worker_activities_per_second,
                             max_tps: config.max_task_queue_activities_per_second,
                         },
                     );
@@ -413,11 +416,11 @@ impl Worker {
                     Some(move |np| np_metrics.record_num_pollers(np)),
                 )) as BoxedNexusPoller;
 
-                #[cfg(test)]
+                #[cfg(any(feature = "test-utilities", test))]
                 let wft_stream = wft_stream.left_stream();
                 (wft_stream, act_poll_buffer, nexus_poll_buffer)
             }
-            #[cfg(test)]
+            #[cfg(any(feature = "test-utilities", test))]
             TaskPollers::Mocked {
                 wft_stream,
                 act_poller,
@@ -483,11 +486,18 @@ impl Worker {
             shutdown_token.child_token(),
         );
 
+        let deployment_options = match &config.versioning_strategy {
+            temporal_sdk_core_api::worker::WorkerVersioningStrategy::WorkerDeploymentBased(
+                opts,
+            ) => Some(opts.clone()),
+            _ => None,
+        };
         let provider = SlotProvider::new(
             config.namespace.clone(),
             config.task_queue.clone(),
             wft_slots.clone(),
             external_wft_tx,
+            deployment_options,
         );
         let worker_key = Mutex::new(client.workers().register(Box::new(provider)));
         let sdk_name_and_ver = client.sdk_name_and_version();
@@ -813,7 +823,7 @@ impl Worker {
     /// Sets a function to be called at the end of each activation completion
     pub(crate) fn set_post_activate_hook(
         &mut self,
-        callback: impl Fn(&Self, PostActivateHookData) + Send + Sync + 'static,
+        callback: impl Fn(&Self, PostActivateHookData<'_>) + Send + Sync + 'static,
     ) {
         self.post_activate_hook = Some(Box::new(callback))
     }
@@ -868,7 +878,7 @@ pub(crate) struct PostActivateHookData<'a> {
 
 pub(crate) enum TaskPollers {
     Real,
-    #[cfg(test)]
+    #[cfg(any(feature = "test-utilities", test))]
     Mocked {
         wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
         act_poller: Option<BoxedPoller<PollActivityTaskQueueResponse>>,

package/sdk-core/core/src/worker/nexus.rs

@@ -32,7 +32,11 @@ use temporal_sdk_core_protos::{
             CancelNexusTask, NexusTask, NexusTaskCancelReason, nexus_task, nexus_task_completion,
         },
     },
-    temporal::api::nexus::v1::{request::Variant, response, start_operation_response},
+    temporal::api::nexus::{
+        self,
+        v1::{request::Variant, response, start_operation_response},
+    },
+    utilities::normalize_http_headers,
 };
 use tokio::{
     join,
@@ -42,7 +46,7 @@ use tokio::{
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tokio_util::sync::CancellationToken;
 
-static REQUEST_TIMEOUT_HEADER: &str = "Request-Timeout";
+static REQUEST_TIMEOUT_HEADER: &str = "request-timeout";
 
 /// Centralizes all state related to received nexus tasks
 pub(super) struct NexusManager {
@@ -245,11 +249,18 @@ where
             .filter_map(move |t| {
                 let res = match t {
                     TaskStreamInput::Poll(t) => match *t {
-                        Ok(t) => {
+                        Ok(mut t) => {
                             if let Some(dur) = t.resp.sched_to_start() {
                                 self.metrics.nexus_task_sched_to_start_latency(dur);
                             };
 
+                            if let Some(ref mut req) = t.resp.request {
+                                req.header = normalize_http_headers(std::mem::take(&mut req.header));
+                                if let Some(nexus::v1::request::Variant::StartOperation(ref mut sor)) = req.variant {
+                                    sor.callback_header = normalize_http_headers(std::mem::take(&mut sor.callback_header));
+                                }
+                            }
+
                             let tt = TaskToken(t.resp.task_token.clone());
                             let mut timeout_task = None;
                             if let Some(timeout_str) = t