@temporalio/core-bridge 1.1.0 → 1.4.0

package/sdk-core/client/src/retry.rs

@@ -1,14 +1,14 @@
 use crate::{
-    ClientOptions, RawClientLikeUser, Result, RetryConfig, WorkflowClientTrait, WorkflowOptions,
-    WorkflowTaskCompletion,
+    ClientOptions, ListClosedFilters, ListOpenFilters, Namespace, Result, RetryConfig,
+    StartTimeFilter, WorkflowClientTrait, WorkflowOptions,
 };
-use backoff::{backoff::Backoff, ExponentialBackoff};
+use backoff::{backoff::Backoff, exponential::ExponentialBackoff, Clock, SystemClock};
 use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
-use std::{fmt::Debug, future::Future, time::Duration};
+use std::{fmt::Debug, future::Future, sync::Arc, time::Duration};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
-        common::v1::{Payload, Payloads},
+        common::v1::{Header, Payload, Payloads},
         enums::v1::WorkflowTaskFailedCause,
         failure::v1::Failure,
         query::v1::WorkflowQuery,
@@ -28,21 +28,26 @@ pub const RETRYABLE_ERROR_CODES: [Code; 7] = [
     Code::OutOfRange,
     Code::Unavailable,
 ];
+const LONG_POLL_FATAL_GRACE: Duration = Duration::from_secs(60);
+/// Must match the method name in [crate::raw::WorkflowService]
+const POLL_WORKFLOW_METH_NAME: &str = "poll_workflow_task_queue";
+/// Must match the method name in [crate::raw::WorkflowService]
+const POLL_ACTIVITY_METH_NAME: &str = "poll_activity_task_queue";
 
 /// A wrapper for a [WorkflowClientTrait] or [crate::WorkflowService] implementor which performs
 /// auto-retries
 #[derive(Debug, Clone)]
 pub struct RetryClient<SG> {
     client: SG,
-    retry_config: RetryConfig,
+    retry_config: Arc<RetryConfig>,
 }
 
 impl<SG> RetryClient<SG> {
     /// Use the provided retry config with the provided client
-    pub const fn new(client: SG, retry_config: RetryConfig) -> Self {
+    pub fn new(client: SG, retry_config: RetryConfig) -> Self {
         Self {
             client,
-            retry_config,
+            retry_config: Arc::new(retry_config),
         }
     }
 }
@@ -81,9 +86,8 @@ impl<SG> RetryClient<SG> {
     }
 
     pub(crate) fn get_retry_config(&self, call_name: &'static str) -> RetryConfig {
-        let call_type = Self::determine_call_type(call_name);
-        match call_type {
-            CallType::Normal => self.retry_config.clone(),
+        match CallType::from_call_name(call_name) {
+            CallType::Normal => (*self.retry_config).clone(),
             CallType::LongPoll => RetryConfig::poll_retry_policy(),
         }
     }
@@ -92,40 +96,39 @@ impl<SG> RetryClient<SG> {
         rtc: RetryConfig,
         factory: F,
         call_name: &'static str,
-    ) -> FutureRetry<F, TonicErrorHandler>
+    ) -> FutureRetry<F, TonicErrorHandler<SystemClock>>
     where
         F: FnMut() -> Fut + Unpin,
         Fut: Future<Output = Result<R>>,
     {
-        let call_type = Self::determine_call_type(call_name);
-        FutureRetry::new(factory, TonicErrorHandler::new(rtc, call_type, call_name))
-    }
-
-    fn determine_call_type(call_name: &str) -> CallType {
-        match call_name {
-            "poll_workflow_task" | "poll_activity_task" => CallType::LongPoll,
-            _ => CallType::Normal,
-        }
+        FutureRetry::new(factory, TonicErrorHandler::new(rtc, call_name))
     }
 }
 
 #[derive(Debug)]
-pub(crate) struct TonicErrorHandler {
-    backoff: ExponentialBackoff,
+pub(crate) struct TonicErrorHandler<C: Clock> {
+    backoff: ExponentialBackoff<C>,
     max_retries: usize,
     call_type: CallType,
     call_name: &'static str,
 }
-impl TonicErrorHandler {
-    fn new(cfg: RetryConfig, call_type: CallType, call_name: &'static str) -> Self {
+impl TonicErrorHandler<SystemClock> {
+    fn new(cfg: RetryConfig, call_name: &'static str) -> Self {
+        Self::new_with_clock(cfg, call_name, SystemClock::default())
+    }
+}
+impl<C> TonicErrorHandler<C>
+where
+    C: Clock,
+{
+    fn new_with_clock(cfg: RetryConfig, call_name: &'static str, clock: C) -> Self {
         Self {
             max_retries: cfg.max_retries,
-            call_type,
+            call_type: CallType::from_call_name(call_name),
             call_name,
-            backoff: cfg.into(),
+            backoff: cfg.into_exp_backoff(clock),
         }
     }
-
     const fn should_log_retry_warning(&self, cur_attempt: usize) -> bool {
         // Warn on more than 5 retries for unlimited retrying
         if self.max_retries == 0 && cur_attempt > 5 {
@@ -144,8 +147,19 @@ pub enum CallType {
     Normal,
     LongPoll,
 }
+impl CallType {
+    fn from_call_name(call_name: &str) -> Self {
+        match call_name {
+            POLL_WORKFLOW_METH_NAME | POLL_ACTIVITY_METH_NAME => CallType::LongPoll,
+            _ => CallType::Normal,
+        }
+    }
+}
 
-impl ErrorHandler<tonic::Status> for TonicErrorHandler {
+impl<C> ErrorHandler<tonic::Status> for TonicErrorHandler<C>
+where
+    C: Clock,
+{
     type OutError = tonic::Status;
 
     fn handle(&mut self, current_attempt: usize, e: tonic::Status) -> RetryPolicy<tonic::Status> {
@@ -154,10 +168,11 @@ impl ErrorHandler<tonic::Status> for TonicErrorHandler {
             return RetryPolicy::ForwardError(e);
         }
 
+        let is_long_poll = self.call_type == CallType::LongPoll;
         // Long polls are OK with being cancelled or running into the timeout because there's
         // nothing to do but retry anyway
-        let long_poll_allowed = self.call_type == CallType::LongPoll
-            && [Code::Cancelled, Code::DeadlineExceeded].contains(&e.code());
+        let long_poll_allowed =
+            is_long_poll && [Code::Cancelled, Code::DeadlineExceeded].contains(&e.code());
 
         if RETRYABLE_ERROR_CODES.contains(&e.code()) || long_poll_allowed {
             if current_attempt == 1 {
@@ -178,6 +193,10 @@ impl ErrorHandler<tonic::Status> for TonicErrorHandler {
                     }
                 }
             }
+        } else if is_long_poll && self.backoff.get_elapsed_time() <= LONG_POLL_FATAL_GRACE {
+            // We permit "fatal" errors while long polling for a while, because some proxies return
+            // stupid error codes while getting ready, among other weird infra issues
+            RetryPolicy::WaitRetry(self.backoff.max_interval)
         } else {
             RetryPolicy::ForwardError(e)
         }
@@ -208,6 +227,7 @@ where
         task_queue: String,
         workflow_id: String,
         workflow_type: String,
+        request_id: Option<String>,
         options: WorkflowOptions,
     ) -> Result<StartWorkflowExecutionResponse> {
         retry_call!(
@@ -217,31 +237,11 @@ where
             task_queue.clone(),
             workflow_id.clone(),
             workflow_type.clone(),
+            request_id.clone(),
             options.clone()
         )
     }
 
-    async fn poll_workflow_task(
-        &self,
-        task_queue: String,
-        is_sticky: bool,
-    ) -> Result<PollWorkflowTaskQueueResponse> {
-        retry_call!(self, poll_workflow_task, task_queue.clone(), is_sticky)
-    }
-
-    async fn poll_activity_task(
-        &self,
-        task_queue: String,
-        max_tasks_per_sec: Option<f64>,
-    ) -> Result<PollActivityTaskQueueResponse> {
-        retry_call!(
-            self,
-            poll_activity_task,
-            task_queue.clone(),
-            max_tasks_per_sec
-        )
-    }
-
     async fn reset_sticky_task_queue(
         &self,
         workflow_id: String,
@@ -255,13 +255,6 @@ where
         )
     }
 
-    async fn complete_workflow_task(
-        &self,
-        request: WorkflowTaskCompletion,
-    ) -> Result<RespondWorkflowTaskCompletedResponse> {
-        retry_call!(self, complete_workflow_task, request.clone())
-    }
-
     async fn complete_activity_task(
         &self,
         task_token: TaskToken,
@@ -335,6 +328,7 @@ where
         run_id: String,
         signal_name: String,
         payloads: Option<Payloads>,
+        request_id: Option<String>,
     ) -> Result<SignalWorkflowExecutionResponse> {
         retry_call!(
             self,
@@ -342,7 +336,8 @@ where
             workflow_id.clone(),
             run_id.clone(),
             signal_name.clone(),
-            payloads.clone()
+            payloads.clone(),
+            request_id.clone()
         )
     }
 
@@ -352,9 +347,11 @@ where
         task_queue: String,
         workflow_id: String,
         workflow_type: String,
+        request_id: Option<String>,
         options: WorkflowOptions,
         signal_name: String,
         signal_input: Option<Payloads>,
+        signal_header: Option<Header>,
     ) -> Result<SignalWithStartWorkflowExecutionResponse> {
         retry_call!(
             self,
@@ -363,9 +360,11 @@ where
             task_queue.clone(),
             workflow_id.clone(),
             workflow_type.clone(),
+            request_id.clone(),
             options.clone(),
             signal_name.clone(),
-            signal_input.clone()
+            signal_input.clone(),
+            signal_header.clone()
         )
     }
 
@@ -430,13 +429,15 @@ where
         workflow_id: String,
         run_id: Option<String>,
         reason: String,
+        request_id: Option<String>,
     ) -> Result<RequestCancelWorkflowExecutionResponse> {
         retry_call!(
             self,
             cancel_workflow_execution,
             workflow_id.clone(),
             run_id.clone(),
-            reason.clone()
+            reason.clone(),
+            request_id.clone()
         )
     }
 
@@ -457,6 +458,59 @@ where
         retry_call!(self, list_namespaces,)
     }
 
+    async fn describe_namespace(&self, namespace: Namespace) -> Result<DescribeNamespaceResponse> {
+        retry_call!(self, describe_namespace, namespace.clone())
+    }
+
+    async fn list_open_workflow_executions(
+        &self,
+        maximum_page_size: i32,
+        next_page_token: Vec<u8>,
+        start_time_filter: Option<StartTimeFilter>,
+        filters: Option<ListOpenFilters>,
+    ) -> Result<ListOpenWorkflowExecutionsResponse> {
+        retry_call!(
+            self,
+            list_open_workflow_executions,
+            maximum_page_size,
+            next_page_token.clone(),
+            start_time_filter.clone(),
+            filters.clone()
+        )
+    }
+
+    async fn list_closed_workflow_executions(
+        &self,
+        maximum_page_size: i32,
+        next_page_token: Vec<u8>,
+        start_time_filter: Option<StartTimeFilter>,
+        filters: Option<ListClosedFilters>,
+    ) -> Result<ListClosedWorkflowExecutionsResponse> {
+        retry_call!(
+            self,
+            list_closed_workflow_executions,
+            maximum_page_size,
+            next_page_token.clone(),
+            start_time_filter.clone(),
+            filters.clone()
+        )
+    }
+
+    async fn list_workflow_executions(
+        &self,
+        page_size: i32,
+        next_page_token: Vec<u8>,
+        query: String,
+    ) -> Result<ListWorkflowExecutionsResponse> {
+        retry_call!(
+            self,
+            list_workflow_executions,
+            page_size,
+            next_page_token.clone(),
+            query.clone()
+        )
+    }
+
     fn get_options(&self) -> &ClientOptions {
         self.client.get_options()
     }
@@ -466,21 +520,13 @@ where
     }
 }
 
-impl<C> RawClientLikeUser for RetryClient<C>
-where
-    C: RawClientLikeUser,
-{
-    type RawClientT = C::RawClientT;
-
-    fn wf_svc(&self) -> Self::RawClientT {
-        self.client.wf_svc()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::MockWorkflowClientTrait;
+    use assert_matches::assert_matches;
+    use backoff::Clock;
+    use std::{ops::Add, time::Instant};
     use tonic::Status;
 
     #[tokio::test]
@@ -510,6 +556,13 @@ mod tests {
         }
     }
 
+    struct FixedClock(Instant);
+    impl Clock for FixedClock {
+        fn now(&self) -> Instant {
+            self.0
+        }
+    }
+
     #[tokio::test]
     async fn long_poll_non_retryable_errors() {
         for code in [
@@ -521,24 +574,48 @@ mod tests {
             Code::Unauthenticated,
             Code::Unimplemented,
         ] {
-            let mut mock_client = MockWorkflowClientTrait::new();
-            mock_client
-                .expect_poll_workflow_task()
-                .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
-                .times(1);
-            mock_client
-                .expect_poll_activity_task()
-                .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
-                .times(1);
-            let retry_client = RetryClient::new(mock_client, Default::default());
-            let result = retry_client
-                .poll_workflow_task("tq".to_string(), false)
-                .await;
-            assert!(result.is_err());
-            let result = retry_client
-                .poll_activity_task("tq".to_string(), None)
-                .await;
-            assert!(result.is_err());
+            for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
+                let retry_cfg = RetryConfig::poll_retry_policy();
+                let mut err_handler = TonicErrorHandler {
+                    max_retries: retry_cfg.max_retries,
+                    call_type: CallType::LongPoll,
+                    call_name,
+                    backoff: retry_cfg.into_exp_backoff(FixedClock(Instant::now())),
+                };
+                let result = err_handler.handle(1, Status::new(code, "Ahh"));
+                assert_matches!(result, RetryPolicy::WaitRetry(_));
+                err_handler.backoff.clock.0 = err_handler
+                    .backoff
+                    .clock
+                    .0
+                    .add(LONG_POLL_FATAL_GRACE + Duration::from_secs(1));
+                let result = err_handler.handle(2, Status::new(code, "Ahh"));
+                assert_matches!(result, RetryPolicy::ForwardError(_));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn long_poll_retryable_errors_never_fatal() {
+        for code in RETRYABLE_ERROR_CODES {
+            for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
+                let retry_cfg = RetryConfig::poll_retry_policy();
+                let mut err_handler = TonicErrorHandler {
+                    max_retries: retry_cfg.max_retries,
+                    call_type: CallType::LongPoll,
+                    call_name,
+                    backoff: retry_cfg.into_exp_backoff(FixedClock(Instant::now())),
+                };
+                let result = err_handler.handle(1, Status::new(code, "Ahh"));
+                assert_matches!(result, RetryPolicy::WaitRetry(_));
+                err_handler.backoff.clock.0 = err_handler
+                    .backoff
+                    .clock
+                    .0
+                    .add(LONG_POLL_FATAL_GRACE + Duration::from_secs(1));
+                let result = err_handler.handle(2, Status::new(code, "Ahh"));
+                assert_matches!(result, RetryPolicy::WaitRetry(_));
+            }
         }
     }
 
@@ -566,68 +643,32 @@ mod tests {
 
     #[tokio::test]
     async fn long_poll_retries_forever() {
-        let mut mock_client = MockWorkflowClientTrait::new();
-        mock_client
-            .expect_poll_workflow_task()
-            .returning(move |_, _| Err(Status::new(Code::Unknown, "retryable failure")))
-            .times(50);
-        mock_client
-            .expect_poll_workflow_task()
-            .returning(|_, _| Ok(Default::default()))
-            .times(1);
-        mock_client
-            .expect_poll_activity_task()
-            .returning(move |_, _| Err(Status::new(Code::Unknown, "retryable failure")))
-            .times(50);
-        mock_client
-            .expect_poll_activity_task()
-            .returning(|_, _| Ok(Default::default()))
-            .times(1);
-
-        let retry_client = RetryClient::new(mock_client, Default::default());
-
-        let result = retry_client
-            .poll_workflow_task("tq".to_string(), false)
-            .await;
-        assert!(result.is_ok());
-        let result = retry_client
-            .poll_activity_task("tq".to_string(), None)
-            .await;
-        assert!(result.is_ok());
+        // A bit odd, but we don't need a real client to test the retry client passes through the
+        // correct retry config
+        let fake_retry = RetryClient::new((), Default::default());
+        for i in 1..=50 {
+            for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
+                let mut err_handler =
+                    TonicErrorHandler::new(fake_retry.get_retry_config(call), call);
+                let result = err_handler.handle(i, Status::new(Code::Unknown, "Ahh"));
+                assert_matches!(result, RetryPolicy::WaitRetry(_));
+            }
+        }
     }
 
     #[tokio::test]
     async fn long_poll_retries_deadline_exceeded() {
+        let fake_retry = RetryClient::new((), Default::default());
         // For some reason we will get cancelled in these situations occasionally (always?) too
         for code in [Code::Cancelled, Code::DeadlineExceeded] {
-            let mut mock_client = MockWorkflowClientTrait::new();
-            mock_client
-                .expect_poll_workflow_task()
-                .returning(move |_, _| Err(Status::new(code, "retryable failure")))
-                .times(5);
-            mock_client
-                .expect_poll_workflow_task()
-                .returning(|_, _| Ok(Default::default()))
-                .times(1);
-            mock_client
-                .expect_poll_activity_task()
-                .returning(move |_, _| Err(Status::new(code, "retryable failure")))
-                .times(5);
-            mock_client
-                .expect_poll_activity_task()
-                .returning(|_, _| Ok(Default::default()))
-                .times(1);
-
-            let retry_client = RetryClient::new(mock_client, Default::default());
-
-            let result = retry_client
-                .poll_workflow_task("tq".to_string(), false)
-                .await;
-            assert!(result.is_ok());
-            let result = retry_client
-                .poll_activity_task("tq".to_string(), None)
-                .await;
-            assert!(result.is_ok());
+            for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
+                let mut err_handler =
+                    TonicErrorHandler::new(fake_retry.get_retry_config(call), call);
+                for i in 1..=5 {
+                    let result = err_handler.handle(i, Status::new(code, "retryable failure"));
+                    assert_matches!(result, RetryPolicy::WaitRetry(_));
+                }
+            }
         }
     }
 }

package/sdk-core/client/src/workflow_handle/mod.rs

@@ -100,7 +100,7 @@ where
             let server_res = self
                 .client
                 .clone()
-                .client()
+                .workflow_client()
                 .get_workflow_execution_history(GetWorkflowExecutionHistoryRequest {
                     namespace: self.info.namespace.to_string(),
                     execution: Some(WorkflowExecution {

package/sdk-core/core/Cargo.toml

@@ -23,39 +23,46 @@ dashmap = "5.0"
 derive_builder = "0.11"
 derive_more = "0.99"
 enum_dispatch = "0.3"
+flate2 = "1.0"
 futures = "0.3"
-governor = "0.4"
+futures-util = "0.3"
+governor = "0.5"
 http = "0.2"
 hyper = "0.14"
 itertools = "0.10"
 lazy_static = "1.4"
 log = "0.4"
-lru = "0.7"
+lru = "0.8"
 mockall = "0.11"
+nix = "0.25"
 once_cell = "1.5"
-opentelemetry = { version = "0.17", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.10.0", features = ["tokio", "metrics"] }
-opentelemetry-prometheus = "0.10.0"
+opentelemetry = { version = "0.18", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.11", features = ["tokio", "metrics"] }
+opentelemetry-prometheus = "0.11"
 parking_lot = { version = "0.12", features = ["send_guard"] }
 prometheus = "0.13"
-prost = "0.9"
-prost-types = "0.9"
+prost = "0.11"
+prost-types = "0.11"
 rand = "0.8.3"
+reqwest = { version = "0.11", features = ["json", "stream", "rustls-tls", "tokio-rustls"], default-features = false }
 ringbuf = "0.2"
 serde = "1.0"
+serde_json = "1.0"
 siphasher = "0.3"
 slotmap = "1.0"
+tar = "0.4"
 thiserror = "1.0"
-tokio = { version = "1.1", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs"] }
-tokio-util = { version = "0.7" }
+tokio = { version = "1.1", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs", "process"] }
+tokio-util = { version = "0.7", features = ["io", "io-util"] }
 tokio-stream = "0.1"
-tonic = { version = "0.6", features = ["tls", "tls-roots"] }
+tonic = { version = "0.8", features = ["tls", "tls-roots"] }
 tracing = { version = "0.1", features = ["log-always"] }
 tracing-futures = "0.2"
-tracing-opentelemetry = "0.17"
+tracing-opentelemetry = "0.18"
 tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter"] }
 url = "2.2"
-uuid = { version = "0.8.2", features = ["v4"] }
+uuid = { version = "1.1", features = ["v4"] }
+zip = "0.6"
 
 # 1st party local deps
 [dependencies.temporal-sdk-core-api]
@@ -78,13 +85,13 @@ version = "0.1"
 [dev-dependencies]
 assert_matches = "1.4"
 bimap = "0.6.1"
-criterion = "0.3"
-rstest = "0.12"
+criterion = "0.4"
+rstest = "0.15"
 temporal-sdk-core-test-utils = { path = "../test-utils" }
 temporal-sdk = { path = "../sdk" }
 
 [build-dependencies]
-tonic-build = "0.6"
+tonic-build = "0.8"
 
 [[test]]
 name = "integ_tests"
@@ -101,3 +108,9 @@ test = false
 [[bench]]
 name = "workflow_replay"
 harness = false
+
+# This is maybe a bit hacky, but we call the runner an "example" because that gets it compiling with
+# the dev-dependencies, which we want.
+[[example]]
+name = "integ_runner"
+path = "../tests/runner.rs"