temporalio 0.0.1 → 0.1.0

data/bridge/sdk-core/client/src/retry.rs

@@ -1,6 +1,6 @@
 use crate::{
-    ClientOptions, ListClosedFilters, ListOpenFilters, Namespace, Result, RetryConfig,
-    StartTimeFilter, WorkflowClientTrait, WorkflowOptions,
+    ClientOptions, ListClosedFilters, ListOpenFilters, Namespace, RegisterNamespaceOptions, Result,
+    RetryConfig, SignalWithStartOptions, StartTimeFilter, WorkflowClientTrait, WorkflowOptions,
 };
 use backoff::{backoff::Backoff, exponential::ExponentialBackoff, Clock, SystemClock};
 use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
@@ -8,7 +8,7 @@ use std::{fmt::Debug, future::Future, sync::Arc, time::Duration};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
-        common::v1::{Header, Payload, Payloads},
+        common::v1::{Payload, Payloads},
         enums::v1::WorkflowTaskFailedCause,
         failure::v1::Failure,
         query::v1::WorkflowQuery,
@@ -101,32 +101,49 @@ impl<SG> RetryClient<SG> {
         F: FnMut() -> Fut + Unpin,
         Fut: Future<Output = Result<R>>,
     {
-        FutureRetry::new(factory, TonicErrorHandler::new(rtc, call_name))
+        FutureRetry::new(
+            factory,
+            TonicErrorHandler::new(rtc, RetryConfig::throttle_retry_policy(), call_name),
+        )
     }
 }
 
 #[derive(Debug)]
 pub(crate) struct TonicErrorHandler<C: Clock> {
     backoff: ExponentialBackoff<C>,
+    throttle_backoff: ExponentialBackoff<C>,
     max_retries: usize,
     call_type: CallType,
     call_name: &'static str,
 }
 impl TonicErrorHandler<SystemClock> {
-    fn new(cfg: RetryConfig, call_name: &'static str) -> Self {
-        Self::new_with_clock(cfg, call_name, SystemClock::default())
+    fn new(cfg: RetryConfig, throttle_cfg: RetryConfig, call_name: &'static str) -> Self {
+        Self::new_with_clock(
+            cfg,
+            throttle_cfg,
+            call_name,
+            SystemClock::default(),
+            SystemClock::default(),
+        )
     }
 }
 impl<C> TonicErrorHandler<C>
 where
     C: Clock,
 {
-    fn new_with_clock(cfg: RetryConfig, call_name: &'static str, clock: C) -> Self {
+    fn new_with_clock(
+        cfg: RetryConfig,
+        throttle_cfg: RetryConfig,
+        call_name: &'static str,
+        clock: C,
+        throttle_clock: C,
+    ) -> Self {
         Self {
             max_retries: cfg.max_retries,
             call_type: CallType::from_call_name(call_name),
             call_name,
             backoff: cfg.into_exp_backoff(clock),
+            throttle_backoff: throttle_cfg.into_exp_backoff(throttle_clock),
         }
     }
     const fn should_log_retry_warning(&self, cur_attempt: usize) -> bool {
@@ -184,10 +201,12 @@ where
             match self.backoff.next_backoff() {
                 None => RetryPolicy::ForwardError(e), // None is returned when we've ran out of time
                 Some(backoff) => {
-                    if cfg!(test) {
-                        // Allow unit tests to do lots of retries quickly. This does *not* apply
-                        // during integration testing, importantly.
-                        RetryPolicy::WaitRetry(Duration::from_millis(1))
+                    // We treat ResourceExhausted as a special case and backoff more
+                    // so we don't overload the server
+                    if e.code() == Code::ResourceExhausted {
+                        let extended_backoff =
+                            backoff.max(self.throttle_backoff.next_backoff().unwrap_or_default());
+                        RetryPolicy::WaitRetry(extended_backoff)
                     } else {
                         RetryPolicy::WaitRetry(backoff)
                     }
@@ -343,28 +362,14 @@ where
 
     async fn signal_with_start_workflow_execution(
         &self,
-        input: Option<Payloads>,
-        task_queue: String,
-        workflow_id: String,
-        workflow_type: String,
-        request_id: Option<String>,
-        options: WorkflowOptions,
-        signal_name: String,
-        signal_input: Option<Payloads>,
-        signal_header: Option<Header>,
+        options: SignalWithStartOptions,
+        workflow_options: WorkflowOptions,
     ) -> Result<SignalWithStartWorkflowExecutionResponse> {
         retry_call!(
             self,
             signal_with_start_workflow_execution,
-            input.clone(),
-            task_queue.clone(),
-            workflow_id.clone(),
-            workflow_type.clone(),
-            request_id.clone(),
             options.clone(),
-            signal_name.clone(),
-            signal_input.clone(),
-            signal_header.clone()
+            workflow_options.clone()
         )
     }
 
@@ -454,6 +459,13 @@ where
         )
     }
 
+    async fn register_namespace(
+        &self,
+        options: RegisterNamespaceOptions,
+    ) -> Result<RegisterNamespaceResponse> {
+        retry_call!(self, register_namespace, options.clone())
+    }
+
     async fn list_namespaces(&self) -> Result<ListNamespacesResponse> {
         retry_call!(self, list_namespaces,)
     }
@@ -511,6 +523,25 @@ where
         )
     }
 
+    async fn list_archived_workflow_executions(
+        &self,
+        page_size: i32,
+        next_page_token: Vec<u8>,
+        query: String,
+    ) -> Result<ListArchivedWorkflowExecutionsResponse> {
+        retry_call!(
+            self,
+            list_archived_workflow_executions,
+            page_size,
+            next_page_token.clone(),
+            query.clone()
+        )
+    }
+
+    async fn get_search_attributes(&self) -> Result<GetSearchAttributesResponse> {
+        retry_call!(self, get_search_attributes)
+    }
+
     fn get_options(&self) -> &ClientOptions {
         self.client.get_options()
     }
@@ -529,6 +560,16 @@ mod tests {
     use std::{ops::Add, time::Instant};
     use tonic::Status;
 
+    /// Predefined retry configs with low durations to make unit tests faster
+    const TEST_RETRY_CONFIG: RetryConfig = RetryConfig {
+        initial_interval: Duration::from_millis(1),
+        randomization_factor: 0.0,
+        multiplier: 1.1,
+        max_interval: Duration::from_millis(2),
+        max_elapsed_time: None,
+        max_retries: 10,
+    };
+
     #[tokio::test]
     async fn non_retryable_errors() {
         for code in [
@@ -547,7 +588,7 @@ mod tests {
                 .expect_cancel_activity_task()
                 .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
                 .times(1);
-            let retry_client = RetryClient::new(mock_client, Default::default());
+            let retry_client = RetryClient::new(mock_client, TEST_RETRY_CONFIG);
             let result = retry_client
                 .cancel_activity_task(vec![1].into(), None)
                 .await;
@@ -575,12 +616,13 @@ mod tests {
             Code::Unimplemented,
         ] {
             for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let retry_cfg = RetryConfig::poll_retry_policy();
                 let mut err_handler = TonicErrorHandler {
-                    max_retries: retry_cfg.max_retries,
+                    max_retries: TEST_RETRY_CONFIG.max_retries,
                     call_type: CallType::LongPoll,
                     call_name,
-                    backoff: retry_cfg.into_exp_backoff(FixedClock(Instant::now())),
+                    backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
+                    throttle_backoff: TEST_RETRY_CONFIG
+                        .into_exp_backoff(FixedClock(Instant::now())),
                 };
                 let result = err_handler.handle(1, Status::new(code, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
@@ -599,12 +641,13 @@ mod tests {
     async fn long_poll_retryable_errors_never_fatal() {
         for code in RETRYABLE_ERROR_CODES {
             for call_name in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let retry_cfg = RetryConfig::poll_retry_policy();
                 let mut err_handler = TonicErrorHandler {
-                    max_retries: retry_cfg.max_retries,
+                    max_retries: TEST_RETRY_CONFIG.max_retries,
                     call_type: CallType::LongPoll,
                     call_name,
-                    backoff: retry_cfg.into_exp_backoff(FixedClock(Instant::now())),
+                    backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
+                    throttle_backoff: TEST_RETRY_CONFIG
+                        .into_exp_backoff(FixedClock(Instant::now())),
                 };
                 let result = err_handler.handle(1, Status::new(code, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
@@ -621,7 +664,12 @@ mod tests {
 
     #[tokio::test]
     async fn retryable_errors() {
-        for code in RETRYABLE_ERROR_CODES {
+        // Take out retry exhausted since it gets a special policy which would make this take ages
+        for code in RETRYABLE_ERROR_CODES
+            .iter()
+            .copied()
+            .filter(|p| p != &Code::ResourceExhausted)
+        {
             let mut mock_client = MockWorkflowClientTrait::new();
             mock_client
                 .expect_cancel_activity_task()
@@ -632,7 +680,7 @@ mod tests {
                 .returning(|_, _| Ok(Default::default()))
                 .times(1);
 
-            let retry_client = RetryClient::new(mock_client, Default::default());
+            let retry_client = RetryClient::new(mock_client, TEST_RETRY_CONFIG);
             let result = retry_client
                 .cancel_activity_task(vec![1].into(), None)
                 .await;
@@ -641,15 +689,53 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn retry_resource_exhausted() {
+        let mut err_handler = TonicErrorHandler {
+            max_retries: TEST_RETRY_CONFIG.max_retries,
+            call_type: CallType::Normal,
+            call_name: POLL_WORKFLOW_METH_NAME,
+            backoff: TEST_RETRY_CONFIG.into_exp_backoff(FixedClock(Instant::now())),
+            throttle_backoff: RetryConfig {
+                initial_interval: Duration::from_millis(2),
+                randomization_factor: 0.0,
+                multiplier: 4.0,
+                max_interval: Duration::from_millis(10),
+                max_elapsed_time: None,
+                max_retries: 10,
+            }
+            .into_exp_backoff(FixedClock(Instant::now())),
+        };
+        let result = err_handler.handle(1, Status::new(Code::ResourceExhausted, "leave me alone"));
+        match result {
+            RetryPolicy::WaitRetry(duration) => assert_eq!(duration, Duration::from_millis(2)),
+            _ => panic!(),
+        }
+        err_handler.backoff.clock.0 = err_handler.backoff.clock.0.add(Duration::from_millis(10));
+        err_handler.throttle_backoff.clock.0 = err_handler
+            .throttle_backoff
+            .clock
+            .0
+            .add(Duration::from_millis(10));
+        let result = err_handler.handle(2, Status::new(Code::ResourceExhausted, "leave me alone"));
+        match result {
+            RetryPolicy::WaitRetry(duration) => assert_eq!(duration, Duration::from_millis(8)),
+            _ => panic!(),
+        }
+    }
+
     #[tokio::test]
     async fn long_poll_retries_forever() {
         // A bit odd, but we don't need a real client to test the retry client passes through the
         // correct retry config
-        let fake_retry = RetryClient::new((), Default::default());
+        let fake_retry = RetryClient::new((), TEST_RETRY_CONFIG);
         for i in 1..=50 {
             for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let mut err_handler =
-                    TonicErrorHandler::new(fake_retry.get_retry_config(call), call);
+                let mut err_handler = TonicErrorHandler::new(
+                    fake_retry.get_retry_config(call),
+                    fake_retry.get_retry_config(call),
+                    call,
+                );
                 let result = err_handler.handle(i, Status::new(Code::Unknown, "Ahh"));
                 assert_matches!(result, RetryPolicy::WaitRetry(_));
             }
@@ -658,12 +744,15 @@ mod tests {
 
     #[tokio::test]
     async fn long_poll_retries_deadline_exceeded() {
-        let fake_retry = RetryClient::new((), Default::default());
+        let fake_retry = RetryClient::new((), TEST_RETRY_CONFIG);
         // For some reason we will get cancelled in these situations occasionally (always?) too
         for code in [Code::Cancelled, Code::DeadlineExceeded] {
             for call in [POLL_WORKFLOW_METH_NAME, POLL_ACTIVITY_METH_NAME] {
-                let mut err_handler =
-                    TonicErrorHandler::new(fake_retry.get_retry_config(call), call);
+                let mut err_handler = TonicErrorHandler::new(
+                    fake_retry.get_retry_config(call),
+                    fake_retry.get_retry_config(call),
+                    call,
+                );
                 for i in 1..=5 {
                     let result = err_handler.handle(i, Status::new(code, "retryable failure"));
                     assert_matches!(result, RetryPolicy::WaitRetry(_));

data/bridge/sdk-core/core/Cargo.toml

@@ -12,15 +12,20 @@ categories = ["development-tools"]
 
 [lib]
 
+[features]
+# Do not enable this feature when building production SDKs. If we ever want a user in the field to
+# record WF input data, we can build them a custom SDK or they can build - it adds significant extra
+# code size in the form of [de]serializers.
+save_wf_inputs = ["rmp-serde", "temporal-sdk-core-protos/serde_serialize"]
+
 [dependencies]
 anyhow = "1.0"
 arc-swap = "1.3"
-async-channel = "1.6"
 async-trait = "0.1"
-base64 = "0.13"
+base64 = "0.21"
 crossbeam = "0.8"
 dashmap = "5.0"
-derive_builder = "0.11"
+derive_builder = "0.12"
 derive_more = "0.99"
 enum_dispatch = "0.3"
 flate2 = "1.0"
@@ -31,21 +36,22 @@ http = "0.2"
 hyper = "0.14"
 itertools = "0.10"
 lazy_static = "1.4"
-log = "0.4"
-lru = "0.8"
+lru = "0.9"
 mockall = "0.11"
-nix = "0.25"
+nix = "0.26"
 once_cell = "1.5"
 opentelemetry = { version = "0.18", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.11", features = ["tokio", "metrics"] }
 opentelemetry-prometheus = "0.11"
 parking_lot = { version = "0.12", features = ["send_guard"] }
+pin-project = "1.0"
 prometheus = "0.13"
 prost = "0.11"
-prost-types = "0.11"
+prost-types = { version = "0.4", package = "prost-wkt-types" }
 rand = "0.8.3"
 reqwest = { version = "0.11", features = ["json", "stream", "rustls-tls", "tokio-rustls"], default-features = false }
-ringbuf = "0.2"
+ringbuf = "0.3"
+rmp-serde = { version = "1.1", optional = true }
 serde = "1.0"
 serde_json = "1.0"
 siphasher = "0.3"
@@ -56,13 +62,13 @@ tokio = { version = "1.1", features = ["rt", "rt-multi-thread", "parking_lot", "
 tokio-util = { version = "0.7", features = ["io", "io-util"] }
 tokio-stream = "0.1"
 tonic = { version = "0.8", features = ["tls", "tls-roots"] }
-tracing = { version = "0.1", features = ["log-always"] }
+tracing = "0.1"
 tracing-futures = "0.2"
 tracing-opentelemetry = "0.18"
-tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter"] }
+tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter", "registry"] }
 url = "2.2"
 uuid = { version = "1.1", features = ["v4"] }
-zip = "0.6"
+zip = "0.6.3"
 
 # 1st party local deps
 [dependencies.temporal-sdk-core-api]
@@ -85,8 +91,9 @@ version = "0.1"
 [dev-dependencies]
 assert_matches = "1.4"
 bimap = "0.6.1"
+clap = { version = "4.0", features = ["derive"] }
 criterion = "0.4"
-rstest = "0.15"
+rstest = "0.16"
 temporal-sdk-core-test-utils = { path = "../test-utils" }
 temporal-sdk = { path = "../sdk" }
 
@@ -101,9 +108,10 @@ path = "../tests/main.rs"
 test = false
 
 [[test]]
-name = "load_tests"
-path = "../tests/load_tests.rs"
+name = "heavy_tests"
+path = "../tests/heavy_tests.rs"
 test = false
+required-features = ["save_wf_inputs"]
 
 [[bench]]
 name = "workflow_replay"
@@ -113,4 +121,9 @@ harness = false
 # the dev-dependencies, which we want.
 [[example]]
 name = "integ_runner"
-path = "../tests/runner.rs"
+path = "../tests/runner.rs"
+
+[[example]]
+name = "wf_input_replay"
+path = "../tests/wf_input_replay.rs"
+required-features = ["save_wf_inputs"]

data/bridge/sdk-core/core/benches/workflow_replay.rs

@@ -1,29 +1,30 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use futures::StreamExt;
 use std::time::Duration;
-use temporal_sdk::{WfContext, Worker, WorkflowFunction};
-use temporal_sdk_core::{telemetry_init, TelemetryOptionsBuilder};
+use temporal_sdk::{WfContext, WorkflowFunction};
+use temporal_sdk_core::replay::HistoryForReplay;
 use temporal_sdk_core_protos::DEFAULT_WORKFLOW_TYPE;
-use temporal_sdk_core_test_utils::{canned_histories, init_core_replay_preloaded};
+use temporal_sdk_core_test_utils::{canned_histories, replay_sdk_worker};
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let tokio_runtime = tokio::runtime::Builder::new_current_thread()
         .enable_time()
         .build()
         .unwrap();
-    telemetry_init(&TelemetryOptionsBuilder::default().build().unwrap()).unwrap();
     let _g = tokio_runtime.enter();
 
     let num_timers = 10;
     let t = canned_histories::long_sequential_timers(num_timers as usize);
-    let hist = t.get_full_history_info().unwrap().into();
+    let hist = HistoryForReplay::new(
+        t.get_full_history_info().unwrap().into(),
+        "whatever".to_string(),
+    );
 
     c.bench_function("Small history replay", |b| {
         b.iter(|| {
             tokio_runtime.block_on(async {
                 let func = timers_wf(num_timers);
-                let (worker, _) = init_core_replay_preloaded("replay_bench", &hist);
-                let mut worker = Worker::new_from_core(worker, "replay_bench".to_string());
+                let mut worker = replay_sdk_worker([hist.clone()]);
                 worker.register_wf(DEFAULT_WORKFLOW_TYPE, func);
                 worker.run().await.unwrap();
             })
@@ -32,14 +33,16 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
     let num_tasks = 50;
     let t = canned_histories::lots_of_big_signals(num_tasks);
-    let hist = t.get_full_history_info().unwrap().into();
+    let hist = HistoryForReplay::new(
+        t.get_full_history_info().unwrap().into(),
+        "whatever".to_string(),
+    );
 
     c.bench_function("Large payloads history replay", |b| {
         b.iter(|| {
             tokio_runtime.block_on(async {
                 let func = big_signals_wf(num_tasks);
-                let (worker, _) = init_core_replay_preloaded("large_hist_bench", &hist);
-                let mut worker = Worker::new_from_core(worker, "large_hist_bench".to_string());
+                let mut worker = replay_sdk_worker([hist.clone()]);
                 worker.register_wf(DEFAULT_WORKFLOW_TYPE, func);
                 worker.run().await.unwrap();
             })