@temporalio/core-bridge 1.11.1 → 1.11.2

package/sdk-core/client/src/retry.rs

@@ -1,10 +1,11 @@
 use crate::{
-    ClientOptions, ListClosedFilters, ListOpenFilters, Namespace, RegisterNamespaceOptions, Result,
-    RetryConfig, SignalWithStartOptions, StartTimeFilter, WorkflowClientTrait, WorkflowOptions,
+    raw::IsUserLongPoll, ClientOptions, ListClosedFilters, ListOpenFilters, Namespace,
+    RegisterNamespaceOptions, Result, RetryConfig, SignalWithStartOptions, StartTimeFilter,
+    WorkflowClientTrait, WorkflowOptions,
 };
 use backoff::{backoff::Backoff, exponential::ExponentialBackoff, Clock, SystemClock};
 use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
-use std::{fmt::Debug, future::Future, sync::Arc, time::Duration};
+use std::{error::Error, fmt::Debug, future::Future, sync::Arc, time::Duration};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
@@ -16,7 +17,7 @@ use temporal_sdk_core_protos::{
     },
     TaskToken,
 };
-use tonic::Code;
+use tonic::{Code, Request};
 
 /// List of gRPC error codes that client will retry.
 pub const RETRYABLE_ERROR_CODES: [Code; 7] = [
@@ -71,6 +72,7 @@ impl<SG> RetryClient<SG> {
     /// Wraps a call to the underlying client with retry capability.
     ///
     /// This is the "old" path used by higher-level [WorkflowClientTrait] implementors
+    // TODO: Get rid of this
     pub(crate) async fn call_with_retry<R, F, Fut>(
         &self,
         factory: F,
@@ -80,22 +82,41 @@ impl<SG> RetryClient<SG> {
         F: Fn() -> Fut + Unpin,
         Fut: Future<Output = Result<R>>,
     {
-        let rtc = self.get_retry_config(call_name);
-        let res = Self::make_future_retry(rtc, factory, call_name).await;
+        let info = self.get_call_info::<()>(call_name, None);
+        let res = Self::make_future_retry(info, factory).await;
         Ok(res.map_err(|(e, _attempt)| e)?.0)
     }
 
-    pub(crate) fn get_retry_config(&self, call_name: &'static str) -> RetryConfig {
-        match CallType::from_call_name(call_name) {
-            CallType::Normal => (*self.retry_config).clone(),
-            CallType::LongPoll => RetryConfig::poll_retry_policy(),
+    pub(crate) fn get_call_info<R>(
+        &self,
+        call_name: &'static str,
+        request: Option<&Request<R>>,
+    ) -> CallInfo {
+        let mut call_type = CallType::Normal;
+        let retry_cfg = {
+            match call_name {
+                POLL_WORKFLOW_METH_NAME | POLL_ACTIVITY_METH_NAME => {
+                    call_type = CallType::LongPoll;
+                    RetryConfig::poll_retry_policy()
+                }
+                _ => (*self.retry_config).clone(),
+            }
+        };
+        if let Some(r) = request.as_ref() {
+            if r.extensions().get::<IsUserLongPoll>().is_some() {
+                call_type = CallType::UserLongPoll;
+            }
+        }
+        CallInfo {
+            call_type,
+            call_name,
+            retry_cfg,
         }
     }
 
     pub(crate) fn make_future_retry<R, F, Fut>(
-        rtc: RetryConfig,
+        info: CallInfo,
         factory: F,
-        call_name: &'static str,
     ) -> FutureRetry<F, TonicErrorHandler<SystemClock>>
     where
         F: FnMut() -> Fut + Unpin,
@@ -103,7 +124,7 @@ impl<SG> RetryClient<SG> {
     {
         FutureRetry::new(
             factory,
-            TonicErrorHandler::new(rtc, RetryConfig::throttle_retry_policy(), call_name),
+            TonicErrorHandler::new(info, RetryConfig::throttle_retry_policy()),
         )
     }
 }
@@ -115,13 +136,13 @@ pub(crate) struct TonicErrorHandler<C: Clock> {
     max_retries: usize,
     call_type: CallType,
     call_name: &'static str,
+    have_retried_goaway_cancel: bool,
 }
 impl TonicErrorHandler<SystemClock> {
-    fn new(cfg: RetryConfig, throttle_cfg: RetryConfig, call_name: &'static str) -> Self {
+    fn new(call_info: CallInfo, throttle_cfg: RetryConfig) -> Self {
         Self::new_with_clock(
-            cfg,
+            call_info,
             throttle_cfg,
-            call_name,
             SystemClock::default(),
             SystemClock::default(),
         )
@@ -132,18 +153,18 @@ where
     C: Clock,
 {
     fn new_with_clock(
-        cfg: RetryConfig,
+        call_info: CallInfo,
         throttle_cfg: RetryConfig,
-        call_name: &'static str,
         clock: C,
         throttle_clock: C,
     ) -> Self {
         Self {
-            max_retries: cfg.max_retries,
-            call_type: CallType::from_call_name(call_name),
-            call_name,
-            backoff: cfg.into_exp_backoff(clock),
+            call_type: call_info.call_type,
+            call_name: call_info.call_name,
+            max_retries: call_info.retry_cfg.max_retries,
+            backoff: call_info.retry_cfg.into_exp_backoff(clock),
             throttle_backoff: throttle_cfg.into_exp_backoff(throttle_clock),
+            have_retried_goaway_cancel: false,
         }
     }
 
@@ -168,19 +189,21 @@ where
         }
     }
 }
+
+#[derive(Clone, Debug, PartialEq)]
+pub(crate) struct CallInfo {
+    call_type: CallType,
+    call_name: &'static str,
+    retry_cfg: RetryConfig,
+}
+
 #[doc(hidden)]
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
 pub enum CallType {
     Normal,
     LongPoll,
-}
-impl CallType {
-    fn from_call_name(call_name: &str) -> Self {
-        match call_name {
-            POLL_WORKFLOW_METH_NAME | POLL_ACTIVITY_METH_NAME => CallType::LongPoll,
-            _ => CallType::Normal,
-        }
-    }
+    // Like a long poll but won't always retry timeouts/cancels
+    UserLongPoll,
 }
 
 impl<C> ErrorHandler<tonic::Status> for TonicErrorHandler<C>
@@ -201,7 +224,26 @@ where
         let long_poll_allowed =
             is_long_poll && [Code::Cancelled, Code::DeadlineExceeded].contains(&e.code());
 
-        if RETRYABLE_ERROR_CODES.contains(&e.code()) || long_poll_allowed {
+        // Sometimes we can get a GOAWAY that, for whatever reason, isn't quite properly handled
+        // by hyper or some other internal lib, and we want to retry that still. We'll retry that
+        // at most once. Ideally this bit should be removed eventually if we can repro the upstream
+        // bug and it is fixed.
+        let mut goaway_retry_allowed = false;
+        if !self.have_retried_goaway_cancel && e.code() == Code::Cancelled {
+            if let Some(e) = e
+                .source()
+                .and_then(|e| e.downcast_ref::<tonic::transport::Error>())
+                .and_then(|te| te.source())
+                .and_then(|tec| tec.downcast_ref::<hyper::Error>())
+            {
+                if format!("{e:?}").contains("connection closed") {
+                    goaway_retry_allowed = true;
+                    self.have_retried_goaway_cancel = true;
+                }
+            }
+        }
+
+        if RETRYABLE_ERROR_CODES.contains(&e.code()) || long_poll_allowed || goaway_retry_allowed {
             if current_attempt == 1 {
                 debug!(error=?e, "gRPC call {} failed on first attempt", self.call_name);
             } else {
@@ -630,14 +672,16 @@ mod tests {
             Code::Unimplemented,
         ] {
             for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let mut err_handler = TonicErrorHandler {
-                    max_retries: TEST_RETRY_CONFIG.max_retries,
-                    call_type: CallType::LongPoll,
-                    call_name,
-                    backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
-                    throttle_backoff: TEST_RETRY_CONFIG
-                        .into_exp_backoff(FixedClock(Instant::now())),
-                };
+                let mut err_handler = TonicErrorHandler::new_with_clock(
+                    CallInfo {
+                        call_type: CallType::LongPoll,
+                        call_name,
+                        retry_cfg: TEST_RETRY_CONFIG,
+                    },
+                    TEST_RETRY_CONFIG,
+                    FixedClock(Instant::now()),
+                    FixedClock(Instant::now()),
+                );
                 let result = err_handler.handle(1, Status::new(code, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
                 err_handler.backoff.clock.0 = err_handler
@@ -655,14 +699,16 @@ mod tests {
     async fn long_poll_retryable_errors_never_fatal() {
         for code in RETRYABLE_ERROR_CODES {
             for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let mut err_handler = TonicErrorHandler {
-                    max_retries: TEST_RETRY_CONFIG.max_retries,
-                    call_type: CallType::LongPoll,
-                    call_name,
-                    backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
-                    throttle_backoff: TEST_RETRY_CONFIG
-                        .into_exp_backoff(FixedClock(Instant::now())),
-                };
+                let mut err_handler = TonicErrorHandler::new_with_clock(
+                    CallInfo {
+                        call_type: CallType::LongPoll,
+                        call_name,
+                        retry_cfg: TEST_RETRY_CONFIG,
+                    },
+                    TEST_RETRY_CONFIG,
+                    FixedClock(Instant::now()),
+                    FixedClock(Instant::now()),
+                );
                 let result = err_handler.handle(1, Status::new(code, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
                 err_handler.backoff.clock.0 = err_handler
@@ -705,21 +751,23 @@ mod tests {
 
     #[tokio::test]
     async fn retry_resource_exhausted() {
-        let mut err_handler = TonicErrorHandler {
-            max_retries: TEST_RETRY_CONFIG.max_retries,
-            call_type: CallType::Normal,
-            call_name: POLL_WORKFLOW_METH_NAME,
-            backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
-            throttle_backoff: RetryConfig {
+        let mut err_handler = TonicErrorHandler::new_with_clock(
+            CallInfo {
+                call_type: CallType::LongPoll,
+                call_name: POLL_WORKFLOW_METH_NAME,
+                retry_cfg: TEST_RETRY_CONFIG,
+            },
+            RetryConfig {
                 initial_interval: Duration::from_millis(2),
                 randomization_factor: 0.0,
                 multiplier: 4.0,
                 max_interval: Duration::from_millis(10),
                 max_elapsed_time: None,
                 max_retries: 10,
-            }
-            .into_exp_backoff(FixedClock(Instant::now())),
-        };
+            },
+            FixedClock(Instant::now()),
+            FixedClock(Instant::now()),
+        );
         let result = err_handler.handle(1, Status::new(Code::ResourceExhausted, "leave me alone"));
         match result {
             RetryPolicy::WaitRetry(duration) => assert_eq!(duration, Duration::from_millis(2)),
@@ -746,9 +794,8 @@ mod tests {
         for i in 1..=50 {
             for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
                 let mut err_handler = TonicErrorHandler::new(
-                    fake_retry.get_retry_config(call),
-                    fake_retry.get_retry_config(call),
-                    call,
+                    fake_retry.get_call_info::<()>(call, None),
+                    RetryConfig::throttle_retry_policy(),
                 );
                 let result = err_handler.handle(i, Status::new(Code::Unknown, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
@@ -763,9 +810,8 @@ mod tests {
         for code in [Code::Cancelled, Code::DeadlineExceeded] {
             for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
                 let mut err_handler = TonicErrorHandler::new(
-                    fake_retry.get_retry_config(call),
-                    fake_retry.get_retry_config(call),
-                    call,
+                    fake_retry.get_call_info::<()>(call, None),
+                    RetryConfig::throttle_retry_policy(),
                 );
                 for i in 1..=5 {
                     let result = err_handler.handle(i, Status::new(code, "retryable failure"));

package/sdk-core/client/src/workflow_handle/mod.rs

@@ -1,4 +1,4 @@
-use crate::{InterceptedMetricsSvc, RawClientLike};
+use crate::{InterceptedMetricsSvc, RawClientLike, WorkflowService};
 use anyhow::{anyhow, bail};
 use std::marker::PhantomData;
 use temporal_sdk_core_protos::{
@@ -107,7 +107,6 @@ where
             let server_res = self
                 .client
                 .clone()
-                .workflow_client_mut()
                 .get_workflow_execution_history(GetWorkflowExecutionHistoryRequest {
                     namespace: self.info.namespace.to_string(),
                     execution: Some(WorkflowExecution {

package/sdk-core/core/Cargo.toml

@@ -40,11 +40,11 @@ hyper = { version = "1.2", optional = true }
 hyper-util = { version = "0.1", features = ["server", "http1", "http2", "tokio"], optional = true }
 itertools = "0.13"
 lru = "0.12"
-mockall = "0.12"
+mockall = "0.13"
 once_cell = { workspace = true }
 opentelemetry = { workspace = true, features = ["metrics"], optional = true }
 opentelemetry_sdk = { version = "0.24", features = ["rt-tokio", "metrics"], optional = true }
-opentelemetry-otlp = { version = "0.17", features = ["tokio", "metrics"], optional = true }
+opentelemetry-otlp = { version = "0.17", features = ["tokio", "metrics", "tls"], optional = true }
 opentelemetry-prometheus = { version = "0.17", optional = true }
 parking_lot = { version = "0.12", features = ["send_guard"] }
 pid = "4.0"
@@ -59,7 +59,7 @@ serde = "1.0"
 serde_json = "1.0"
 siphasher = "1.0"
 slotmap = "1.0"
-sysinfo = "0.30"
+sysinfo = { version = "0.31", default-features = false, features = ["system"] }
 tar = { version = "0.4", optional = true }
 thiserror = "1.0"
 tokio = { version = "1.37", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs", "process"] }
@@ -92,8 +92,7 @@ assert_matches = "1.4"
 bimap = "0.6.1"
 clap = { version = "4.0", features = ["derive"] }
 criterion = "0.5"
-rstest = "0.21"
-sysinfo = "0.30"
+rstest = "0.22"
 temporal-sdk-core-test-utils = { path = "../test-utils" }
 temporal-sdk = { path = "../sdk" }
 tokio-stream = { version = "0.1", features = ["net"] }

package/sdk-core/core/src/abstractions.rs

@@ -3,7 +3,6 @@
 pub(crate) mod take_cell;
 
 use crate::MetricsContext;
-use derive_more::DebugCustom;
 use std::{
     fmt::{Debug, Formatter},
     sync::{
@@ -217,9 +216,9 @@ where
 }
 
 /// Tracks an OwnedMeteredSemPermit and calls on_drop when dropped.
-#[derive(DebugCustom)]
+#[derive(derive_more::Debug)]
+#[debug("Tracked({inner:?})")]
 #[clippy::has_significant_drop]
-#[debug(fmt = "Tracked({inner:?})")]
 pub(crate) struct TrackedOwnedMeteredSemPermit<SK: SlotKind> {
     inner: Option<OwnedMeteredSemPermit<SK>>,
     on_drop: Box<dyn Fn() + Send + Sync>,

package/sdk-core/core/src/core_tests/activity_tasks.rs

@@ -462,7 +462,7 @@ async fn activity_timeout_no_double_resolve() {
         WorkflowCachingPolicy::NonSticky,
         &[
             gen_assert_and_reply(
-                &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
+                &job_assert!(workflow_activation_job::Variant::InitializeWorkflow(_)),
                 vec![ScheduleActivity {
                     seq: activity_id,
                     activity_id: activity_id.to_string(),

package/sdk-core/core/src/core_tests/local_activities.rs

@@ -636,7 +636,7 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
         assert_matches!(
             task.jobs.as_slice(),
             &[WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+                variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
             },]
         );
         core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
@@ -1239,7 +1239,7 @@ async fn queries_can_be_received_while_heartbeating() {
     assert_matches!(
         task.jobs.as_slice(),
         &[WorkflowActivationJob {
-            variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+            variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
         },]
     );
     core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(

package/sdk-core/core/src/core_tests/queries.rs

@@ -272,6 +272,7 @@ async fn legacy_query_failure_on_wft_failure() {
             message: "Ahh i broke".to_string(),
             ..Default::default()
         },
+        None,
     ))
     .await
     .unwrap();
@@ -312,11 +313,14 @@ async fn query_failure_because_nondeterminism(#[values(true, false)] legacy: boo
         mock.num_expected_fails = 1;
     }
     let mut mock = build_mock_pollers(mock);
-    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    mock.worker_cfg(|wc| {
+        wc.max_cached_workflows = 10;
+        wc.ignore_evicts_on_shutdown = false;
+    });
     let core = mock_worker(mock);
 
     let task = core.poll_workflow_activation().await.unwrap();
-    // Nondeterminism, should result in WFT/query being failed
+    // Nondeterminism, should result in WFT being failed
     core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
         .await
         .unwrap();
@@ -474,7 +478,7 @@ async fn query_cache_miss_causes_page_fetch_dont_reply_wft_too_early(
     assert_matches!(
         task.jobs.as_slice(),
         [WorkflowActivationJob {
-            variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+            variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
         }]
     );
     core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
@@ -696,7 +700,7 @@ async fn new_query_fail() {
     assert_matches!(
         task.jobs[0],
         WorkflowActivationJob {
-            variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+            variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
         }
     );
 

package/sdk-core/core/src/core_tests/updates.rs

@@ -46,10 +46,10 @@ async fn replay_with_empty_first_task() {
         task.jobs.as_slice(),
         [
             WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::DoUpdate(_)),
+                variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
             },
             WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+                variant: Some(workflow_activation_job::Variant::DoUpdate(_)),
             },
         ]
     );

package/sdk-core/core/src/core_tests/workflow_cancels.rs

@@ -58,7 +58,7 @@ async fn timer_then_cancel_req(
         NonSticky,
         &[
             gen_assert_and_reply(
-                &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
+                &job_assert!(workflow_activation_job::Variant::InitializeWorkflow(_)),
                 vec![start_timer_cmd(timer_seq, Duration::from_secs(1))],
             ),
             gen_assert_and_reply(
@@ -84,7 +84,7 @@ async fn timer_then_cancel_req_then_timer_then_cancelled() {
         NonSticky,
         &[
             gen_assert_and_reply(
-                &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
+                &job_assert!(workflow_activation_job::Variant::InitializeWorkflow(_)),
                 vec![start_timer_cmd(1, Duration::from_secs(1))],
             ),
             gen_assert_and_reply(
@@ -114,7 +114,7 @@ async fn immediate_cancel() {
         NonSticky,
         &[gen_assert_and_reply(
             &job_assert!(
-                workflow_activation_job::Variant::StartWorkflow(_),
+                workflow_activation_job::Variant::InitializeWorkflow(_),
                 workflow_activation_job::Variant::CancelWorkflow(_)
             ),
             vec![CancelWorkflowExecution {}.into()],