@temporalio/core-bridge 0.16.4 → 0.18.0

package/sdk-core/core/src/retry_logic.rs

@@ -0,0 +1,144 @@
+use std::time::Duration;
+use temporal_sdk_core_protos::{coresdk::common::RetryPolicy, utilities::TryIntoOrNone};
+
+pub(crate) trait RetryPolicyExt {
+    /// Ask this retry policy if a retry should be performed. Caller provides the current attempt
+    /// number - the first attempt should start at 1.
+    ///
+    /// Returns `None` if it should not, otherwise a duration indicating how long to wait before
+    /// performing the retry.
+    fn should_retry(&self, attempt_number: usize, err_type_str: &str) -> Option<Duration>;
+}
+
+impl RetryPolicyExt for RetryPolicy {
+    fn should_retry(&self, attempt_number: usize, err_type_str: &str) -> Option<Duration> {
+        let realmax = self.maximum_attempts.max(0);
+        if realmax > 0 && attempt_number >= realmax as usize {
+            return None;
+        }
+
+        for pat in &self.non_retryable_error_types {
+            if err_type_str.to_lowercase() == pat.to_lowercase() {
+                return None;
+            }
+        }
+
+        let converted_interval = self.initial_interval.clone().try_into_or_none();
+        if attempt_number == 1 {
+            return converted_interval;
+        }
+        let coeff = if self.backoff_coefficient != 0. {
+            self.backoff_coefficient
+        } else {
+            2.0
+        };
+
+        if let Some(interval) = converted_interval {
+            let max_iv = self
+                .maximum_interval
+                .clone()
+                .try_into_or_none()
+                .unwrap_or_else(|| interval.saturating_mul(100));
+            let mul_factor = coeff.powi(attempt_number as i32 - 1);
+            let tried_mul = try_from_secs_f64(mul_factor * interval.as_secs_f64());
+            Some(tried_mul.unwrap_or(max_iv).min(max_iv))
+        } else {
+            // No retries if initial interval is not specified
+            None
+        }
+    }
+}
+
+const NANOS_PER_SEC: u32 = 1_000_000_000;
+/// modified from rust stdlib since this feature is currently nightly only
+fn try_from_secs_f64(secs: f64) -> Option<Duration> {
+    const MAX_NANOS_F64: f64 = ((u64::MAX as u128 + 1) * (NANOS_PER_SEC as u128)) as f64;
+    let nanos = secs * (NANOS_PER_SEC as f64);
+    if !nanos.is_finite() || nanos >= MAX_NANOS_F64 || nanos < 0.0 {
+        None
+    } else {
+        Some(Duration::from_secs_f64(secs))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn calcs_backoffs_properly() {
+        let rp = RetryPolicy {
+            initial_interval: Some(Duration::from_secs(1).into()),
+            backoff_coefficient: 2.0,
+            maximum_interval: Some(Duration::from_secs(10).into()),
+            maximum_attempts: 10,
+            non_retryable_error_types: vec![],
+        };
+        let res = rp.should_retry(1, "").unwrap();
+        assert_eq!(res.as_millis(), 1_000);
+        let res = rp.should_retry(2, "").unwrap();
+        assert_eq!(res.as_millis(), 2_000);
+        let res = rp.should_retry(3, "").unwrap();
+        assert_eq!(res.as_millis(), 4_000);
+        let res = rp.should_retry(4, "").unwrap();
+        assert_eq!(res.as_millis(), 8_000);
+        let res = rp.should_retry(5, "").unwrap();
+        assert_eq!(res.as_millis(), 10_000);
+        let res = rp.should_retry(6, "").unwrap();
+        assert_eq!(res.as_millis(), 10_000);
+        // Max attempts - no retry
+        assert!(rp.should_retry(10, "").is_none());
+    }
+
+    #[test]
+    fn no_interval_no_backoff() {
+        let rp = RetryPolicy {
+            initial_interval: None,
+            backoff_coefficient: 2.0,
+            maximum_interval: None,
+            maximum_attempts: 10,
+            non_retryable_error_types: vec![],
+        };
+        assert!(rp.should_retry(1, "").is_none());
+    }
+
+    #[test]
+    fn max_attempts_zero_retry_forever() {
+        let rp = RetryPolicy {
+            initial_interval: Some(Duration::from_secs(1).into()),
+            backoff_coefficient: 1.2,
+            maximum_interval: None,
+            maximum_attempts: 0,
+            non_retryable_error_types: vec![],
+        };
+        for i in 0..50 {
+            assert!(rp.should_retry(i, "").is_some());
+        }
+    }
+
+    #[test]
+    fn no_overflows() {
+        let rp = RetryPolicy {
+            initial_interval: Some(Duration::from_secs(1).into()),
+            backoff_coefficient: 10.,
+            maximum_interval: None,
+            maximum_attempts: 0,
+            non_retryable_error_types: vec![],
+        };
+        for i in 0..50 {
+            assert!(rp.should_retry(i, "").is_some());
+        }
+    }
+
+    #[test]
+    fn no_retry_err_str_match() {
+        let rp = RetryPolicy {
+            initial_interval: Some(Duration::from_secs(1).into()),
+            backoff_coefficient: 2.0,
+            maximum_interval: Some(Duration::from_secs(10).into()),
+            maximum_attempts: 10,
+            non_retryable_error_types: vec!["no retry".to_string()],
+        };
+        assert!(rp.should_retry(1, "no retry").is_none());
+    }
+}

package/sdk-core/{src → core/src}/telemetry/metrics.rs

@@ -20,15 +20,11 @@ pub(crate) struct MetricsContext {
     //   of lifetime issues: https://github.com/open-telemetry/opentelemetry-rust/issues/629
     //   Use once fixed.
     kvs: Arc<Vec<KeyValue>>,
-    poll_is_long: bool,
 }
 
 impl MetricsContext {
     fn new(kvs: Vec<KeyValue>) -> Self {
-        Self {
-            kvs: Arc::new(kvs),
-            poll_is_long: false,
-        }
+        Self { kvs: Arc::new(kvs) }
     }
 
     pub(crate) fn top_level(namespace: String) -> Self {
@@ -39,14 +35,7 @@ impl MetricsContext {
     pub(crate) fn with_new_attrs(&self, new_kvs: impl IntoIterator<Item = KeyValue>) -> Self {
         let mut kvs = self.kvs.clone();
         Arc::make_mut(&mut kvs).extend(new_kvs);
-        Self {
-            kvs,
-            poll_is_long: false,
-        }
-    }
-
-    pub(crate) fn set_is_long_poll(&mut self) {
-        self.poll_is_long = true;
+        Self { kvs }
     }
 
     /// A workflow task queue poll succeeded
@@ -149,37 +138,10 @@ impl MetricsContext {
     pub(crate) fn cache_size(&self, size: u64) {
         STICKY_CACHE_SIZE.record(size, &self.kvs);
     }
-
-    /// A request to the temporal service was made
-    pub(crate) fn svc_request(&self) {
-        if self.poll_is_long {
-            LONG_SVC_REQUEST.add(1, &self.kvs);
-        } else {
-            SVC_REQUEST.add(1, &self.kvs);
-        }
-    }
-
-    /// A request to the temporal service failed
-    pub(crate) fn svc_request_failed(&self) {
-        if self.poll_is_long {
-            LONG_SVC_REQUEST_FAILED.add(1, &self.kvs);
-        } else {
-            SVC_REQUEST_FAILED.add(1, &self.kvs);
-        }
-    }
-
-    /// Record service request latency
-    pub(crate) fn record_svc_req_latency(&self, dur: Duration) {
-        if self.poll_is_long {
-            LONG_SVC_REQUEST_LATENCY.record(dur.as_millis() as u64, &self.kvs);
-        } else {
-            SVC_REQUEST_LATENCY.record(dur.as_millis() as u64, &self.kvs);
-        }
-    }
 }
 
 lazy_static::lazy_static! {
-    static ref METRIC_METER: Meter = {
+    pub(crate) static ref METRIC_METER: Meter = {
         #[cfg(not(test))]
         if crate::telemetry::GLOBAL_TELEM_DAT.get().is_none() {
             panic!("Tried to use a metric but telemetry has not been initialized")
@@ -212,7 +174,6 @@ const KEY_WF_TYPE: &str = "workflow_type";
 const KEY_TASK_QUEUE: &str = "task_queue";
 const KEY_ACT_TYPE: &str = "activity_type";
 const KEY_POLLER_TYPE: &str = "poller_type";
-const KEY_SVC_METHOD: &str = "operation";
 
 pub(crate) fn workflow_poller() -> KeyValue {
     KeyValue::new(KEY_POLLER_TYPE, "workflow_task")
@@ -232,9 +193,6 @@ pub(crate) fn activity_type(ty: String) -> KeyValue {
 pub(crate) fn workflow_type(ty: String) -> KeyValue {
     KeyValue::new(KEY_WF_TYPE, ty)
 }
-pub(crate) fn svc_operation(op: String) -> KeyValue {
-    KeyValue::new(KEY_SVC_METHOD, op)
-}
 
 tm!(ctr, WF_COMPLETED_COUNTER, "workflow_completed");
 tm!(ctr, WF_CANCELED_COUNTER, "workflow_canceled");
@@ -296,19 +254,6 @@ tm!(ctr, STICKY_CACHE_MISS, "sticky_cache_miss");
 const STICKY_CACHE_SIZE_NAME: &str = "sticky_cache_size";
 tm!(vr_u64, STICKY_CACHE_SIZE, STICKY_CACHE_SIZE_NAME);
 
-tm!(ctr, SVC_REQUEST, "request");
-tm!(ctr, SVC_REQUEST_FAILED, "request_failure");
-const SVC_REQUEST_LATENCY_NAME: &str = "request_latency";
-tm!(vr_u64, SVC_REQUEST_LATENCY, SVC_REQUEST_LATENCY_NAME);
-tm!(ctr, LONG_SVC_REQUEST, "long_request");
-tm!(ctr, LONG_SVC_REQUEST_FAILED, "long_request_failure");
-const LONG_SVC_REQUEST_LATENCY_NAME: &str = "long_request_latency";
-tm!(
-    vr_u64,
-    LONG_SVC_REQUEST_LATENCY,
-    LONG_SVC_REQUEST_LATENCY_NAME
-);
-
 /// Artisanal, handcrafted latency buckets for workflow e2e latency which should expose a useful
 /// set of buckets for < 1 day runtime workflows. Beyond that, this metric probably isn't very
 /// helpful

package/sdk-core/{src → core/src}/telemetry/mod.rs

@@ -66,7 +66,7 @@ impl Default for TelemetryOptions {
 
 /// Things that need to not be dropped while telemetry is ongoing
 #[derive(Default)]
-struct GlobalTelemDat {
+pub(crate) struct GlobalTelemDat {
     metric_push_controller: Option<PushController>,
     core_export_logger: Option<CoreExportLogger>,
     runtime: Option<tokio::runtime::Runtime>,
@@ -92,14 +92,16 @@ impl GlobalTelemDat {
 /// called more than once, subsequent calls do nothing.
 ///
 /// See [TelemetryOptions] docs for more on configuration.
-pub(crate) fn telemetry_init(opts: &TelemetryOptions) -> Result<(), anyhow::Error> {
+pub(crate) fn telemetry_init(
+    opts: &TelemetryOptions,
+) -> Result<&'static GlobalTelemDat, anyhow::Error> {
     // TODO: Per-layer filtering has been implemented but does not yet support
     //   env-filter. When it does, allow filtering logs/telemetry separately.
 
     // Ensure we don't pointlessly spawn threads that won't do anything or call telem dat's init 2x
     let guard = TELETM_MUTEX.lock();
-    if GLOBAL_TELEM_DAT.get().is_some() {
-        return Ok(());
+    if let Some(gtd) = GLOBAL_TELEM_DAT.get() {
+        return Ok(gtd);
     }
 
     // This is a bit odd, but functional. It's desirable to create a separate tokio runtime for
@@ -202,12 +204,10 @@ pub(crate) fn telemetry_init(opts: &TelemetryOptions) -> Result<(), anyhow::Erro
         })?;
 
         res.init();
-        Result::<_, anyhow::Error>::Ok(())
+        Result::<_, anyhow::Error>::Ok(res)
     })
     .join()
-    .expect("Telemetry initialization panicked")?;
-
-    Ok(())
+    .expect("Telemetry initialization panicked")
 }
 
 /// Returned buffered logs for export to lang from the global logging instance

package/sdk-core/{src → core/src}/test_help/mod.rs

@@ -1,43 +1,35 @@
-pub mod canned_histories;
-
-mod history_builder;
-mod history_info;
-
-pub(crate) use history_builder::{TestHistoryBuilder, DEFAULT_WORKFLOW_TYPE};
+pub(crate) use temporal_sdk_core_test_utils::canned_histories;
 
 use crate::{
-    pollers::{
-        BoxedActPoller, BoxedPoller, BoxedWFPoller, MockManualPoller, MockPoller,
-        MockServerGatewayApis,
-    },
-    task_token::TaskToken,
+    pollers::{BoxedActPoller, BoxedPoller, BoxedWFPoller, MockManualPoller, MockPoller},
+    replay::TestHistoryBuilder,
     workflow::WorkflowCachingPolicy,
-    Core, CoreInitOptionsBuilder, CoreSDK, ServerGatewayApis, ServerGatewayOptions, Url,
-    WorkerConfig, WorkerConfigBuilder,
+    Core, CoreInitOptionsBuilder, CoreSDK, ServerGatewayApis, TaskToken, WorkerConfig,
+    WorkerConfigBuilder,
 };
 use bimap::BiMap;
 use futures::FutureExt;
 use mockall::TimesRange;
 use parking_lot::RwLock;
-use rand::{thread_rng, Rng};
 use std::{
     collections::{BTreeMap, HashMap, HashSet, VecDeque},
     ops::RangeFull,
-    str::FromStr,
     sync::Arc,
 };
+use temporal_client::{
+    mocks::{fake_sg_opts, mock_gateway},
+    MockServerGatewayApis,
+};
 use temporal_sdk_core_protos::{
     coresdk::{
-        workflow_activation::WfActivation,
+        workflow_activation::WorkflowActivation,
         workflow_commands::workflow_command,
-        workflow_completion::{self, wf_activation_completion, WfActivationCompletion},
+        workflow_completion::{self, workflow_activation_completion, WorkflowActivationCompletion},
     },
     temporal::api::{
-        common::v1::{WorkflowExecution, WorkflowType},
-        enums::v1::{TaskQueueKind, WorkflowTaskFailedCause},
+        common::v1::WorkflowExecution,
+        enums::v1::WorkflowTaskFailedCause,
         failure::v1::Failure,
-        history::v1::History,
-        taskqueue::v1::TaskQueue,
         workflowservice::v1::{
             PollActivityTaskQueueResponse, PollWorkflowTaskQueueResponse,
             RespondWorkflowTaskCompletedResponse,
@@ -45,7 +37,6 @@ use temporal_sdk_core_protos::{
     },
 };
 
-pub type Result<T, E = anyhow::Error> = std::result::Result<T, E>;
 pub const TEST_Q: &str = "q";
 pub static NO_MORE_WORK_ERROR_MSG: &str = "No more work to do";
 
@@ -146,6 +137,7 @@ pub struct FakeWfResponses {
 pub struct MocksHolder<SG> {
     sg: SG,
     mock_pollers: HashMap<String, MockWorker>,
+    // bidirectional mapping of run id / task token
     pub outstanding_task_map: Option<Arc<RwLock<BiMap<String, TaskToken>>>>,
 }
 
@@ -169,11 +161,7 @@ pub struct MockWorker {
 
 impl Default for MockWorker {
     fn default() -> Self {
-        Self {
-            wf_poller: Box::from(mock_poller()),
-            act_poller: None,
-            config: WorkerConfig::default_test_q(),
-        }
+        Self::for_queue(TEST_Q)
     }
 }
 
@@ -182,15 +170,15 @@ impl MockWorker {
         Self {
             wf_poller,
             act_poller: None,
-            config: WorkerConfig::default(q),
+            config: WorkerConfigBuilder::default()
+                .task_queue(q)
+                .build()
+                .unwrap(),
         }
     }
+
     pub fn for_queue(q: &str) -> Self {
-        Self {
-            wf_poller: Box::from(mock_poller()),
-            act_poller: None,
-            config: WorkerConfig::default(q),
-        }
+        Self::new(q, Box::from(mock_poller()))
     }
 }
 
@@ -335,7 +323,7 @@ impl MockPollCfg {
             hists,
             enforce_correct_number_of_polls,
             num_expected_fails,
-            mock_gateway: MockServerGatewayApis::new(),
+            mock_gateway: mock_gateway(),
             expect_fail_wft_matcher: Box::new(|_, _, _| true),
         }
     }
@@ -381,16 +369,6 @@ pub fn build_mock_pollers(mut cfg: MockPollCfg) -> MocksHolder<MockServerGateway
             }
         }
 
-        // TODO: Fix -- or not? Sticky invalidation could make this pointless anyway
-        // Verify response batches only ever return longer histories (IE: Are sorted ascending)
-        // assert!(
-        //     hist.response_batches
-        //         .as_slice()
-        //         .windows(2)
-        //         .all(|w| w[0] <= w[1]),
-        //     "response batches must have increasing wft numbers"
-        // );
-
         if cfg.enforce_correct_number_of_polls {
             *correct_num_polls.get_or_insert(0) += hist.response_batches.len();
         }
@@ -502,40 +480,15 @@ pub fn hist_to_poll_resp(
         ResponseType::OneTask(tn) => t.get_one_wft(tn).unwrap(),
         ResponseType::AllHistory => t.get_full_history_info().unwrap(),
     };
-    let batch = hist_info.events().to_vec();
-    let task_token: [u8; 16] = thread_rng().gen();
-    PollWorkflowTaskQueueResponse {
-        history: Some(History { events: batch }),
-        workflow_execution: Some(wf),
-        task_token: task_token.to_vec(),
-        workflow_type: Some(WorkflowType {
-            name: DEFAULT_WORKFLOW_TYPE.to_owned(),
-        }),
-        workflow_execution_task_queue: Some(TaskQueue {
-            name: task_queue,
-            kind: TaskQueueKind::Normal as i32,
-        }),
-        previous_started_event_id: hist_info.previous_started_event_id,
-        started_event_id: hist_info.workflow_task_started_event_id,
-        ..Default::default()
-    }
-}
-
-pub fn fake_sg_opts() -> ServerGatewayOptions {
-    ServerGatewayOptions {
-        target_url: Url::from_str("https://fake").unwrap(),
-        namespace: "".to_string(),
-        client_name: "".to_string(),
-        client_version: "".to_string(),
-        static_headers: Default::default(),
-        identity: "".to_string(),
-        worker_binary_id: "".to_string(),
-        tls_cfg: None,
-        retry_config: Default::default(),
-    }
+    let mut resp = hist_info.as_poll_wft_response(task_queue);
+    resp.workflow_execution = Some(wf);
+    resp
 }
 
-type AsserterWithReply<'a> = (&'a dyn Fn(&WfActivation), wf_activation_completion::Status);
+type AsserterWithReply<'a> = (
+    &'a dyn Fn(&WorkflowActivation),
+    workflow_activation_completion::Status,
+);
 
 /// This function accepts a list of asserts and replies to workflow activations to run against the
 /// provided instance of fake core.
@@ -601,10 +554,10 @@ pub(crate) async fn poll_and_reply_clears_outstanding_evicts<'a>(
 
             let reply = if res.jobs.is_empty() {
                 // Just an eviction
-                WfActivationCompletion::empty(TEST_Q, res.run_id.clone())
+                WorkflowActivationCompletion::empty(TEST_Q, res.run_id.clone())
             } else {
                 // Eviction plus some work, we still want to issue the reply
-                WfActivationCompletion {
+                WorkflowActivationCompletion {
                     task_queue: TEST_Q.to_string(),
                     run_id: res.run_id.clone(),
                     status: Some(reply.clone()),
@@ -638,12 +591,11 @@ pub(crate) async fn poll_and_reply_clears_outstanding_evicts<'a>(
     }
 
     assert_eq!(expected_fail_count, executed_failures.len());
-    // TODO: Really need a worker abstraction for testing
-    // assert_eq!(core.wft_manager.outstanding_wft(), 0);
+    assert_eq!(core.outstanding_wfts(TEST_Q), 0);
 }
 
 pub(crate) fn gen_assert_and_reply(
-    asserter: &dyn Fn(&WfActivation),
+    asserter: &dyn Fn(&WorkflowActivation),
     reply_commands: Vec<workflow_command::Variant>,
 ) -> AsserterWithReply<'_> {
     (
@@ -652,7 +604,7 @@ pub(crate) fn gen_assert_and_reply(
     )
 }
 
-pub(crate) fn gen_assert_and_fail(asserter: &dyn Fn(&WfActivation)) -> AsserterWithReply<'_> {
+pub(crate) fn gen_assert_and_fail(asserter: &dyn Fn(&WorkflowActivation)) -> AsserterWithReply<'_> {
     (
         asserter,
         workflow_completion::Failure {
@@ -672,7 +624,7 @@ macro_rules! job_assert {
         |res| {
             assert_matches!(
                 res.jobs.as_slice(),
-                [$(WfActivationJob {
+                [$(WorkflowActivationJob {
                     variant: Some($pat),
                 }),+]
             );